From c1ce9a5d36dcc07bd734b307e7a3a3fb39ebd61d Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Thu, 11 Jan 2024 12:59:30 +0530 Subject: [PATCH 01/64] removed registerInArgo multiple impls --- Wire.go | 2 - .../argocdServer/ArgoClientWrapperService.go | 37 ++++++++---- internal/util/ChartDeploymentService.go | 38 ------------ .../AppStoreDeploymentFullModeService.go | 17 +----- .../gitops/AppStoreDeploymentArgoCdService.go | 8 +-- pkg/chart/ChartService.go | 15 ----- pkg/chart/mocks/ChartService.go | 13 ----- .../DeploymentPipelineConfigService.go | 58 +++++++------------ wire_gen.go | 5 +- 9 files changed, 55 insertions(+), 138 deletions(-) delete mode 100644 internal/util/ChartDeploymentService.go diff --git a/Wire.go b/Wire.go index 9cc5e42f2f..6918016d1f 100644 --- a/Wire.go +++ b/Wire.go @@ -264,8 +264,6 @@ func InitializeApp() (*App, error) { wire.Bind(new(restHandler.MigrateDbRestHandler), new(*restHandler.MigrateDbRestHandlerImpl)), util.NewChartTemplateServiceImpl, wire.Bind(new(util.ChartTemplateService), new(*util.ChartTemplateServiceImpl)), - util.NewChartDeploymentServiceImpl, - wire.Bind(new(util.ChartDeploymentService), new(*util.ChartDeploymentServiceImpl)), //scoped variables start variables.NewScopedVariableServiceImpl, diff --git a/client/argocdServer/ArgoClientWrapperService.go b/client/argocdServer/ArgoClientWrapperService.go index 3cd2cf7380..add0a58122 100644 --- a/client/argocdServer/ArgoClientWrapperService.go +++ b/client/argocdServer/ArgoClientWrapperService.go @@ -3,10 +3,12 @@ package argocdServer import ( "context" application2 "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" + repository2 "github.com/argoproj/argo-cd/v2/pkg/apiclient/repository" "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" "github.com/caarlos0/env" "github.com/devtron-labs/devtron/client/argocdServer/application" "github.com/devtron-labs/devtron/client/argocdServer/bean" + "github.com/devtron-labs/devtron/client/argocdServer/repository" "go.uber.org/zap" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -34,22 +36,25 @@ type ArgoClientWrapperService interface { // UpdateArgoCDSyncModeIfNeeded - if ARGO_AUTO_SYNC_ENABLED=true and app is in manual sync mode or vice versa update app UpdateArgoCDSyncModeIfNeeded(ctx context.Context, argoApplication *v1alpha1.Application) (err error) + + //RegisterGitOpsRepoInArgo - register a repository in argo-cd + RegisterGitOpsRepoInArgo(ctx context.Context, repoUrl string) (err error) } type ArgoClientWrapperServiceImpl struct { - logger *zap.SugaredLogger - acdClient application.ServiceClient - ACDConfig *ACDConfig + logger *zap.SugaredLogger + acdClient application.ServiceClient + ACDConfig *ACDConfig + repositoryService repository.ServiceClient } -func NewArgoClientWrapperServiceImpl(logger *zap.SugaredLogger, - acdClient application.ServiceClient, - ACDConfig *ACDConfig, -) *ArgoClientWrapperServiceImpl { +func NewArgoClientWrapperServiceImpl(logger *zap.SugaredLogger, acdClient application.ServiceClient, + ACDConfig *ACDConfig, repositoryService repository.ServiceClient) *ArgoClientWrapperServiceImpl { return &ArgoClientWrapperServiceImpl{ - logger: logger, - acdClient: acdClient, - ACDConfig: ACDConfig, + logger: logger, + acdClient: acdClient, + ACDConfig: ACDConfig, + repositoryService: repositoryService, } } @@ -130,3 +135,15 @@ func (impl *ArgoClientWrapperServiceImpl) CreateRequestForArgoCDSyncModeUpdateRe Retry: argoApplication.Spec.SyncPolicy.Retry, }}} } + +func (impl *ArgoClientWrapperServiceImpl) RegisterGitOpsRepoInArgo(ctx context.Context, repoUrl string) (err error) { + repo := &v1alpha1.Repository{ + Repo: repoUrl, + } + repo, err = impl.repositoryService.Create(ctx, &repository2.RepoCreateRequest{Repo: repo, Upsert: true}) + if err != nil { + impl.logger.Errorw("error in creating argo Repository ", "err", err) + } + impl.logger.Infow("gitOps repo registered in argo", "name", repoUrl) + return err +} diff --git a/internal/util/ChartDeploymentService.go b/internal/util/ChartDeploymentService.go deleted file mode 100644 index 53dd03deb1..0000000000 --- a/internal/util/ChartDeploymentService.go +++ /dev/null @@ -1,38 +0,0 @@ -package util - -import ( - "context" - repository3 "github.com/argoproj/argo-cd/v2/pkg/apiclient/repository" - "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" - repository4 "github.com/devtron-labs/devtron/client/argocdServer/repository" - "go.uber.org/zap" -) - -type ChartDeploymentService interface { - RegisterInArgo(chartGitAttribute *ChartGitAttribute, ctx context.Context) error -} - -type ChartDeploymentServiceImpl struct { - logger *zap.SugaredLogger - repositoryService repository4.ServiceClient -} - -func NewChartDeploymentServiceImpl(logger *zap.SugaredLogger, repositoryService repository4.ServiceClient) *ChartDeploymentServiceImpl { - return &ChartDeploymentServiceImpl{ - logger: logger, - repositoryService: repositoryService, - } -} - -func (impl *ChartDeploymentServiceImpl) RegisterInArgo(chartGitAttribute *ChartGitAttribute, ctx context.Context) error { - repo := &v1alpha1.Repository{ - Repo: chartGitAttribute.RepoUrl, - } - repo, err := impl.repositoryService.Create(ctx, &repository3.RepoCreateRequest{Repo: repo, Upsert: true}) - if err != nil { - impl.logger.Errorw("error in creating argo Repository ", "err", err) - return err - } - impl.logger.Infow("repo registered in argo", "name", chartGitAttribute.RepoUrl) - return err -} diff --git a/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go b/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go index 0899e6d7d0..179c59463b 100644 --- a/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go +++ b/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go @@ -45,8 +45,6 @@ import ( "github.com/go-pg/pg" "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" - repository2 "github.com/argoproj/argo-cd/v2/pkg/apiclient/repository" - "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" application2 "github.com/devtron-labs/devtron/client/argocdServer/application" "github.com/devtron-labs/devtron/client/argocdServer/repository" "github.com/devtron-labs/devtron/internal/util" @@ -64,7 +62,6 @@ const ( type AppStoreDeploymentFullModeService interface { AppStoreDeployOperationGIT(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, *util.ChartGitAttribute, error) AppStoreDeployOperationACD(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, chartGitAttr *util.ChartGitAttribute, ctx context.Context, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) - RegisterInArgo(chartGitAttribute *util.ChartGitAttribute, ctx context.Context) error SyncACD(acdAppName string, ctx context.Context) UpdateValuesYaml(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) UpdateRequirementYaml(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, appStoreAppVersion *appStoreDiscoverRepository.AppStoreApplicationVersion) error @@ -323,7 +320,7 @@ func (impl AppStoreDeploymentFullModeServiceImpl) AppStoreDeployOperationACD(ins ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) defer cancel() //STEP 4: registerInArgo - err := impl.RegisterInArgo(chartGitAttr, ctx) + err := impl.argoClientWrapperService.RegisterGitOpsRepoInArgo(ctx, chartGitAttr.RepoUrl) if err != nil { impl.logger.Errorw("error in argo registry", "err", err) return nil, err @@ -366,18 +363,6 @@ func (impl AppStoreDeploymentFullModeServiceImpl) AppStoreDeployOperationACD(ins return installAppVersionRequest, nil } -func (impl AppStoreDeploymentFullModeServiceImpl) RegisterInArgo(chartGitAttribute *util.ChartGitAttribute, ctx context.Context) error { - repo := &v1alpha1.Repository{ - Repo: chartGitAttribute.RepoUrl, - } - repo, err := impl.repositoryService.Create(ctx, &repository2.RepoCreateRequest{Repo: repo, Upsert: true}) - if err != nil { - impl.logger.Errorw("error in creating argo Repository ", "err", err) - } - impl.logger.Debugw("repo registered in argo", "name", chartGitAttribute.RepoUrl) - return err -} - func (impl AppStoreDeploymentFullModeServiceImpl) SyncACD(acdAppName string, ctx context.Context) { req := new(application.ApplicationSyncRequest) req.Name = &acdAppName diff --git a/pkg/appStore/deployment/tool/gitops/AppStoreDeploymentArgoCdService.go b/pkg/appStore/deployment/tool/gitops/AppStoreDeploymentArgoCdService.go index 143a4e437b..3b39254642 100644 --- a/pkg/appStore/deployment/tool/gitops/AppStoreDeploymentArgoCdService.go +++ b/pkg/appStore/deployment/tool/gitops/AppStoreDeploymentArgoCdService.go @@ -74,9 +74,9 @@ type AppStoreDeploymentArgoCdServiceImpl struct { helmAppService client.HelmAppService gitOpsConfigRepository repository3.GitOpsConfigRepository appStatusService appStatus.AppStatusService - pipelineStatusTimelineService status.PipelineStatusTimelineService - userService user.UserService - pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository + pipelineStatusTimelineService status.PipelineStatusTimelineService + userService user.UserService + pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository argoClientWrapperService argocdServer.ArgoClientWrapperService acdConfig *argocdServer.ACDConfig @@ -662,7 +662,7 @@ func (impl AppStoreDeploymentArgoCdServiceImpl) patchAcdApp(ctx context.Context, ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) defer cancel() //registerInArgo - err := impl.appStoreDeploymentFullModeService.RegisterInArgo(chartGitAttr, ctx) + err := impl.argoClientWrapperService.RegisterGitOpsRepoInArgo(ctx, chartGitAttr.RepoUrl) if err != nil { impl.Logger.Errorw("error in argo registry", "err", err) return nil, err diff --git a/pkg/chart/ChartService.go b/pkg/chart/ChartService.go index a728aef48a..c95a5d05ec 100644 --- a/pkg/chart/ChartService.go +++ b/pkg/chart/ChartService.go @@ -50,8 +50,6 @@ import ( "github.com/devtron-labs/devtron/pkg/sql" dirCopy "github.com/otiai10/copy" - repository2 "github.com/argoproj/argo-cd/v2/pkg/apiclient/repository" - "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" "github.com/devtron-labs/devtron/client/argocdServer/repository" "github.com/devtron-labs/devtron/internal/sql/models" repository3 "github.com/devtron-labs/devtron/internal/sql/repository" @@ -91,7 +89,6 @@ type ChartService interface { FormatChartName(chartName string) string ValidateUploadedFileFormat(fileName string) error ReadChartMetaDataForLocation(chartDir string, fileName string) (*ChartYamlStruct, error) - RegisterInArgo(chartGitAttribute *util.ChartGitAttribute, ctx context.Context) error FetchCustomChartsInfo() ([]*ChartDto, error) CheckCustomChartByAppId(id int) (bool, error) CheckCustomChartByChartId(id int) (bool, error) @@ -587,18 +584,6 @@ func (impl ChartServiceImpl) CreateChartFromEnvOverride(templateRequest Template return chartVal, err } -func (impl ChartServiceImpl) RegisterInArgo(chartGitAttribute *util.ChartGitAttribute, ctx context.Context) error { - repo := &v1alpha1.Repository{ - Repo: chartGitAttribute.RepoUrl, - } - repo, err := impl.repositoryService.Create(ctx, &repository2.RepoCreateRequest{Repo: repo, Upsert: true}) - if err != nil { - impl.logger.Errorw("error in creating argo Repository ", "err", err) - } - impl.logger.Infow("repo registered in argo", "name", chartGitAttribute.RepoUrl) - return err -} - // converts db object to bean func (impl ChartServiceImpl) chartAdaptor(chart *chartRepoRepository.Chart, appLevelMetrics *repository3.AppLevelMetrics) (*TemplateRequest, error) { var appMetrics bool diff --git a/pkg/chart/mocks/ChartService.go b/pkg/chart/mocks/ChartService.go index 1c58d68537..3cf2e392d6 100644 --- a/pkg/chart/mocks/ChartService.go +++ b/pkg/chart/mocks/ChartService.go @@ -680,19 +680,6 @@ func (_m *ChartService) ReadChartMetaDataForLocation(chartDir string, fileName s return r0, r1 } -// RegisterInArgo provides a mock function with given fields: chartGitAttribute, ctx -func (_m *ChartService) RegisterInArgo(chartGitAttribute *util.ChartGitAttribute, ctx context.Context) error { - ret := _m.Called(chartGitAttribute, ctx) - - var r0 error - if rf, ok := ret.Get(0).(func(*util.ChartGitAttribute, context.Context) error); ok { - r0 = rf(chartGitAttribute, ctx) - } else { - r0 = ret.Error(0) - } - - return r0 -} // UpdateAppOverride provides a mock function with given fields: ctx, templateRequest func (_m *ChartService) UpdateAppOverride(ctx context.Context, templateRequest *chart.TemplateRequest) (*chart.TemplateRequest, error) { diff --git a/pkg/pipeline/DeploymentPipelineConfigService.go b/pkg/pipeline/DeploymentPipelineConfigService.go index 865dc9c82e..ca0c9df270 100644 --- a/pkg/pipeline/DeploymentPipelineConfigService.go +++ b/pkg/pipeline/DeploymentPipelineConfigService.go @@ -26,6 +26,7 @@ import ( bean2 "github.com/devtron-labs/devtron/api/bean" client "github.com/devtron-labs/devtron/api/helm-app" models2 "github.com/devtron-labs/devtron/api/helm-app/models" + "github.com/devtron-labs/devtron/client/argocdServer" "github.com/devtron-labs/devtron/client/argocdServer/application" "github.com/devtron-labs/devtron/internal/sql/models" "github.com/devtron-labs/devtron/internal/sql/repository" @@ -132,7 +133,6 @@ type CdPipelineConfigServiceImpl struct { pipelineStrategyHistoryService history.PipelineStrategyHistoryService chartRepository chartRepoRepository.ChartRepository resourceGroupService resourceGroup2.ResourceGroupService - chartDeploymentService util.ChartDeploymentService chartTemplateService util.ChartTemplateService propertiesConfigService PropertiesConfigService appLevelMetricsRepository repository.AppLevelMetricsRepository @@ -145,42 +145,26 @@ type CdPipelineConfigServiceImpl struct { devtronAppCMCSService DevtronAppCMCSService ciPipelineConfigService CiPipelineConfigService buildPipelineSwitchService BuildPipelineSwitchService + argoClientWrapperService argocdServer.ArgoClientWrapperService } -func NewCdPipelineConfigServiceImpl( - logger *zap.SugaredLogger, - pipelineRepository pipelineConfig.PipelineRepository, - environmentRepository repository2.EnvironmentRepository, - pipelineConfigRepository chartConfig.PipelineConfigRepository, - appWorkflowRepository appWorkflow.AppWorkflowRepository, - pipelineStageService PipelineStageService, - appRepo app2.AppRepository, - appService app.AppService, - deploymentGroupRepository repository.DeploymentGroupRepository, - ciCdPipelineOrchestrator CiCdPipelineOrchestrator, - appStatusRepository appStatus.AppStatusRepository, - ciPipelineRepository pipelineConfig.CiPipelineRepository, - prePostCdScriptHistoryService history.PrePostCdScriptHistoryService, - clusterRepository repository2.ClusterRepository, - helmAppService client.HelmAppService, - enforcerUtil rbac.EnforcerUtil, - gitOpsRepository repository.GitOpsConfigRepository, +func NewCdPipelineConfigServiceImpl(logger *zap.SugaredLogger, pipelineRepository pipelineConfig.PipelineRepository, + environmentRepository repository2.EnvironmentRepository, pipelineConfigRepository chartConfig.PipelineConfigRepository, + appWorkflowRepository appWorkflow.AppWorkflowRepository, pipelineStageService PipelineStageService, + appRepo app2.AppRepository, appService app.AppService, deploymentGroupRepository repository.DeploymentGroupRepository, + ciCdPipelineOrchestrator CiCdPipelineOrchestrator, appStatusRepository appStatus.AppStatusRepository, + ciPipelineRepository pipelineConfig.CiPipelineRepository, prePostCdScriptHistoryService history.PrePostCdScriptHistoryService, + clusterRepository repository2.ClusterRepository, helmAppService client.HelmAppService, + enforcerUtil rbac.EnforcerUtil, gitOpsRepository repository.GitOpsConfigRepository, pipelineStrategyHistoryService history.PipelineStrategyHistoryService, - chartRepository chartRepoRepository.ChartRepository, - resourceGroupService resourceGroup2.ResourceGroupService, - chartDeploymentService util.ChartDeploymentService, - chartTemplateService util.ChartTemplateService, - propertiesConfigService PropertiesConfigService, - appLevelMetricsRepository repository.AppLevelMetricsRepository, - deploymentTemplateHistoryService history.DeploymentTemplateHistoryService, - scopedVariableManager variables.ScopedVariableManager, - deploymentConfig *DeploymentServiceTypeConfig, - application application.ServiceClient, - customTagService CustomTagService, - pipelineConfigListenerService PipelineConfigListenerService, - devtronAppCMCSService DevtronAppCMCSService, - ciPipelineConfigService CiPipelineConfigService, - buildPipelineSwitchService BuildPipelineSwitchService) *CdPipelineConfigServiceImpl { + chartRepository chartRepoRepository.ChartRepository, resourceGroupService resourceGroup2.ResourceGroupService, + chartTemplateService util.ChartTemplateService, propertiesConfigService PropertiesConfigService, + appLevelMetricsRepository repository.AppLevelMetricsRepository, deploymentTemplateHistoryService history.DeploymentTemplateHistoryService, + scopedVariableManager variables.ScopedVariableManager, deploymentConfig *DeploymentServiceTypeConfig, + application application.ServiceClient, customTagService CustomTagService, + pipelineConfigListenerService PipelineConfigListenerService, devtronAppCMCSService DevtronAppCMCSService, + ciPipelineConfigService CiPipelineConfigService, buildPipelineSwitchService BuildPipelineSwitchService, + argoClientWrapperService argocdServer.ArgoClientWrapperService) *CdPipelineConfigServiceImpl { return &CdPipelineConfigServiceImpl{ logger: logger, pipelineRepository: pipelineRepository, @@ -202,7 +186,6 @@ func NewCdPipelineConfigServiceImpl( pipelineStrategyHistoryService: pipelineStrategyHistoryService, chartRepository: chartRepository, resourceGroupService: resourceGroupService, - chartDeploymentService: chartDeploymentService, chartTemplateService: chartTemplateService, propertiesConfigService: propertiesConfigService, appLevelMetricsRepository: appLevelMetricsRepository, @@ -215,6 +198,7 @@ func NewCdPipelineConfigServiceImpl( customTagService: customTagService, ciPipelineConfigService: ciPipelineConfigService, buildPipelineSwitchService: buildPipelineSwitchService, + argoClientWrapperService: argoClientWrapperService, } } @@ -1616,7 +1600,7 @@ func (impl *CdPipelineConfigServiceImpl) ValidateCDPipelineRequest(pipelineCreat func (impl *CdPipelineConfigServiceImpl) RegisterInACD(gitOpsRepoName string, chartGitAttr *util.ChartGitAttribute, userId int32, ctx context.Context) error { - err := impl.chartDeploymentService.RegisterInArgo(chartGitAttr, ctx) + err := impl.argoClientWrapperService.RegisterGitOpsRepoInArgo(ctx, chartGitAttr.RepoUrl) if err != nil { impl.logger.Errorw("error while register git repo in argo", "err", err) emptyRepoErrorMessage := []string{"failed to get index: 404 Not Found", "remote repository is empty"} @@ -1628,7 +1612,7 @@ func (impl *CdPipelineConfigServiceImpl) RegisterInACD(gitOpsRepoName string, ch return err } // - retry register in argo - err = impl.chartDeploymentService.RegisterInArgo(chartGitAttr, ctx) + err = impl.argoClientWrapperService.RegisterGitOpsRepoInArgo(ctx, chartGitAttr.RepoUrl) if err != nil { impl.logger.Errorw("error in re-try register in argo", "err", err) return err diff --git a/wire_gen.go b/wire_gen.go index 1ec6b644e1..600ede7536 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -415,7 +415,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - argoClientWrapperServiceImpl := argocdServer.NewArgoClientWrapperServiceImpl(sugaredLogger, applicationServiceClientImpl, acdConfig) + argoClientWrapperServiceImpl := argocdServer.NewArgoClientWrapperServiceImpl(sugaredLogger, applicationServiceClientImpl, acdConfig, repositoryServiceClientImpl) appStoreDeploymentFullModeServiceImpl := appStoreDeploymentFullMode.NewAppStoreDeploymentFullModeServiceImpl(sugaredLogger, chartTemplateServiceImpl, refChartProxyDir, repositoryServiceClientImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, applicationServiceClientImpl, argoK8sClientImpl, gitFactory, acdAuthConfig, globalEnvVariables, installedAppRepositoryImpl, tokenCache, argoUserServiceImpl, gitOpsConfigRepositoryImpl, pipelineStatusTimelineServiceImpl, appStoreDeploymentCommonServiceImpl, argoClientWrapperServiceImpl, pubSubClientServiceImpl, installedAppVersionHistoryRepositoryImpl, acdConfig) appStoreDeploymentArgoCdServiceImpl := appStoreDeploymentGitopsTool.NewAppStoreDeploymentArgoCdServiceImpl(sugaredLogger, appStoreDeploymentFullModeServiceImpl, applicationServiceClientImpl, chartGroupDeploymentRepositoryImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, chartTemplateServiceImpl, gitFactory, argoUserServiceImpl, appStoreDeploymentCommonServiceImpl, helmAppServiceImpl, gitOpsConfigRepositoryImpl, appStatusServiceImpl, pipelineStatusTimelineServiceImpl, userServiceImpl, pipelineStatusTimelineRepositoryImpl, appStoreApplicationVersionRepositoryImpl, argoClientWrapperServiceImpl, acdConfig) deploymentServiceTypeConfig, err := service.GetDeploymentServiceTypeConfig() @@ -499,14 +499,13 @@ func InitializeApp() (*App, error) { ciMaterialConfigServiceImpl := pipeline.NewCiMaterialConfigServiceImpl(sugaredLogger, materialRepositoryImpl, ciTemplateServiceImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, gitMaterialHistoryServiceImpl, pipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl) imageTaggingRepositoryImpl := repository13.NewImageTaggingRepositoryImpl(db) imageTaggingServiceImpl := pipeline.NewImageTaggingServiceImpl(imageTaggingRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, sugaredLogger) - chartDeploymentServiceImpl := util.NewChartDeploymentServiceImpl(sugaredLogger, repositoryServiceClientImpl) propertiesConfigServiceImpl := pipeline.NewPropertiesConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, chartRefRepositoryImpl, utilMergeUtil, environmentRepositoryImpl, ciCdPipelineOrchestratorImpl, applicationServiceClientImpl, envLevelAppMetricsRepositoryImpl, appLevelMetricsRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl) pipelineDeploymentServiceTypeConfig, err := pipeline.GetDeploymentServiceTypeConfig() if err != nil { return nil, err } devtronAppCMCSServiceImpl := pipeline.NewDevtronAppCMCSServiceImpl(sugaredLogger, appServiceImpl, attributesRepositoryImpl) - cdPipelineConfigServiceImpl := pipeline.NewCdPipelineConfigServiceImpl(sugaredLogger, pipelineRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, appRepositoryImpl, appServiceImpl, deploymentGroupRepositoryImpl, ciCdPipelineOrchestratorImpl, appStatusRepositoryImpl, ciPipelineRepositoryImpl, prePostCdScriptHistoryServiceImpl, clusterRepositoryImpl, helmAppServiceImpl, enforcerUtilImpl, gitOpsConfigRepositoryImpl, pipelineStrategyHistoryServiceImpl, chartRepositoryImpl, resourceGroupServiceImpl, chartDeploymentServiceImpl, chartTemplateServiceImpl, propertiesConfigServiceImpl, appLevelMetricsRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, pipelineDeploymentServiceTypeConfig, applicationServiceClientImpl, customTagServiceImpl, pipelineConfigListenerServiceImpl, devtronAppCMCSServiceImpl, ciPipelineConfigServiceImpl, buildPipelineSwitchServiceImpl) + cdPipelineConfigServiceImpl := pipeline.NewCdPipelineConfigServiceImpl(sugaredLogger, pipelineRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, appRepositoryImpl, appServiceImpl, deploymentGroupRepositoryImpl, ciCdPipelineOrchestratorImpl, appStatusRepositoryImpl, ciPipelineRepositoryImpl, prePostCdScriptHistoryServiceImpl, clusterRepositoryImpl, helmAppServiceImpl, enforcerUtilImpl, gitOpsConfigRepositoryImpl, pipelineStrategyHistoryServiceImpl, chartRepositoryImpl, resourceGroupServiceImpl, chartTemplateServiceImpl, propertiesConfigServiceImpl, appLevelMetricsRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, pipelineDeploymentServiceTypeConfig, applicationServiceClientImpl, customTagServiceImpl, pipelineConfigListenerServiceImpl, devtronAppCMCSServiceImpl, ciPipelineConfigServiceImpl, buildPipelineSwitchServiceImpl, argoClientWrapperServiceImpl) appArtifactManagerImpl := pipeline.NewAppArtifactManagerImpl(sugaredLogger, cdWorkflowRepositoryImpl, userServiceImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, ciWorkflowRepositoryImpl, pipelineStageServiceImpl, cdPipelineConfigServiceImpl, dockerArtifactStoreRepositoryImpl, ciPipelineRepositoryImpl, ciTemplateServiceImpl) globalStrategyMetadataChartRefMappingRepositoryImpl := chartRepoRepository.NewGlobalStrategyMetadataChartRefMappingRepositoryImpl(db, sugaredLogger) devtronAppStrategyServiceImpl := pipeline.NewDevtronAppStrategyServiceImpl(sugaredLogger, chartRepositoryImpl, globalStrategyMetadataChartRefMappingRepositoryImpl, ciCdPipelineOrchestratorImpl, cdPipelineConfigServiceImpl) From b8fa93b0fdc71242c618a8486413448c1df8fc93 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Thu, 11 Jan 2024 15:18:45 +0530 Subject: [PATCH 02/64] extracted app metrics code --- Wire.go | 4 + api/restHandler/CoreAppRestHandler.go | 19 +- .../app/DeploymentPipelineRestHandler.go | 44 ---- api/router/PipelineConfigRouter.go | 1 - pkg/app/AppService.go | 13 +- pkg/chart/ChartService.go | 197 ++++-------------- .../deployedAppMetrics/DeployedAppMetrics.go | 117 +++++++++++ .../manifest/deployedAppMetrics/bean/bean.go | 9 + .../DeploymentTemplateService.go | 16 ++ pkg/pipeline/DeploymentConfigService.go | 18 +- .../DeploymentPipelineConfigService.go | 20 +- pkg/pipeline/WorkflowDagExecutor.go | 20 +- .../DeploymentTemplateHistoryService.go | 15 +- wire_gen.go | 18 +- 14 files changed, 235 insertions(+), 276 deletions(-) create mode 100644 pkg/deployment/manifest/deployedAppMetrics/DeployedAppMetrics.go create mode 100644 pkg/deployment/manifest/deployedAppMetrics/bean/bean.go create mode 100644 pkg/deployment/manifest/deploymentTemplate/DeploymentTemplateService.go diff --git a/Wire.go b/Wire.go index 6918016d1f..b4ef34b994 100644 --- a/Wire.go +++ b/Wire.go @@ -97,6 +97,7 @@ import ( chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" "github.com/devtron-labs/devtron/pkg/commonService" delete2 "github.com/devtron-labs/devtron/pkg/delete" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" "github.com/devtron-labs/devtron/pkg/deploymentGroup" "github.com/devtron-labs/devtron/pkg/devtronResource" repository9 "github.com/devtron-labs/devtron/pkg/devtronResource/repository" @@ -289,6 +290,9 @@ func InitializeApp() (*App, error) { //end + deployedAppMetrics.NewDeployedAppMetricsServiceImpl, + wire.Bind(new(deployedAppMetrics.DeployedAppMetricsService), new(*deployedAppMetrics.DeployedAppMetricsServiceImpl)), + chart.NewChartServiceImpl, wire.Bind(new(chart.ChartService), new(*chart.ChartServiceImpl)), bulkAction.NewBulkUpdateServiceImpl, diff --git a/api/restHandler/CoreAppRestHandler.go b/api/restHandler/CoreAppRestHandler.go index 20566db8c4..6d7f20171f 100644 --- a/api/restHandler/CoreAppRestHandler.go +++ b/api/restHandler/CoreAppRestHandler.go @@ -75,9 +75,9 @@ type CoreAppRestHandler interface { } type CoreAppRestHandlerImpl struct { - logger *zap.SugaredLogger - userAuthService user.UserService - validator *validator.Validate + logger *zap.SugaredLogger + userAuthService user.UserService + validator *validator.Validate enforcerUtil rbac.EnforcerUtil enforcer casbin.Enforcer appCrudOperationService app.AppCrudOperationService @@ -1391,19 +1391,6 @@ func (handler CoreAppRestHandlerImpl) createDeploymentTemplate(ctx context.Conte handler.logger.Errorw("service err, Create in CreateDeploymentTemplate", "err", err, "createRequest", createDeploymentTemplateRequest) return err, http.StatusInternalServerError } - - //updating app metrics - appMetricsRequest := chart.AppMetricEnableDisableRequest{ - AppId: appId, - UserId: userId, - IsAppMetricsEnabled: deploymentTemplate.ShowAppMetrics, - } - _, err = handler.chartService.AppMetricsEnableDisable(appMetricsRequest) - if err != nil { - handler.logger.Errorw("service err, AppMetricsEnableDisable in createDeploymentTemplate", "err", err, "appId", appId, "payload", appMetricsRequest) - return err, http.StatusInternalServerError - } - return nil, http.StatusOK } diff --git a/api/restHandler/app/DeploymentPipelineRestHandler.go b/api/restHandler/app/DeploymentPipelineRestHandler.go index 8eedcc34b5..c7dc4f7da6 100644 --- a/api/restHandler/app/DeploymentPipelineRestHandler.go +++ b/api/restHandler/app/DeploymentPipelineRestHandler.go @@ -80,7 +80,6 @@ type DevtronAppDeploymentConfigRestHandler interface { GetDeploymentPipelineStrategy(w http.ResponseWriter, r *http.Request) GetDefaultDeploymentPipelineStrategy(w http.ResponseWriter, r *http.Request) - AppMetricsEnableDisable(w http.ResponseWriter, r *http.Request) EnvMetricsEnableDisable(w http.ResponseWriter, r *http.Request) EnvConfigOverrideCreateNamespace(w http.ResponseWriter, r *http.Request) @@ -1700,49 +1699,6 @@ func (handler PipelineConfigRestHandlerImpl) EnvConfigOverrideReset(w http.Respo common.WriteJsonResp(w, err, isSuccess, http.StatusOK) } -func (handler PipelineConfigRestHandlerImpl) AppMetricsEnableDisable(w http.ResponseWriter, r *http.Request) { - decoder := json.NewDecoder(r.Body) - userId, err := handler.userAuthService.GetLoggedInUser(r) - if userId == 0 || err != nil { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return - } - token := r.Header.Get("token") - vars := mux.Vars(r) - appId, err := strconv.Atoi(vars["appId"]) - if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - var appMetricEnableDisableRequest chart.AppMetricEnableDisableRequest - err = decoder.Decode(&appMetricEnableDisableRequest) - appMetricEnableDisableRequest.AppId = appId - appMetricEnableDisableRequest.UserId = userId - if err != nil { - handler.Logger.Errorw("request err, AppMetricsEnableDisable", "err", err, "appId", appId, "payload", appMetricEnableDisableRequest) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - handler.Logger.Infow("request payload, AppMetricsEnableDisable", "err", err, "appId", appId, "payload", appMetricEnableDisableRequest) - app, err := handler.pipelineBuilder.GetApp(appId) - if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - resourceName := handler.enforcerUtil.GetAppRBACName(app.AppName) - if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, resourceName); !ok { - common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden) - return - } - createResp, err := handler.chartService.AppMetricsEnableDisable(appMetricEnableDisableRequest) - if err != nil { - handler.Logger.Errorw("service err, AppMetricsEnableDisable", "err", err, "appId", appId, "payload", appMetricEnableDisableRequest) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - common.WriteJsonResp(w, err, createResp, http.StatusOK) -} - func (handler PipelineConfigRestHandlerImpl) EnvMetricsEnableDisable(w http.ResponseWriter, r *http.Request) { userId, err := handler.userAuthService.GetLoggedInUser(r) if userId == 0 || err != nil { diff --git a/api/router/PipelineConfigRouter.go b/api/router/PipelineConfigRouter.go index 54bc7613ec..2fb48b3d8d 100644 --- a/api/router/PipelineConfigRouter.go +++ b/api/router/PipelineConfigRouter.go @@ -134,7 +134,6 @@ func (router PipelineConfigRouterImpl) initPipelineConfigRouter(configRouter *mu configRouter.Path("/env/reset/{appId}/{environmentId}/{id}").HandlerFunc(router.restHandler.EnvConfigOverrideReset).Methods("DELETE") configRouter.Path("/env/namespace/{appId}/{environmentId}").HandlerFunc(router.restHandler.EnvConfigOverrideCreateNamespace).Methods("POST") - configRouter.Path("/template/metrics/{appId}").HandlerFunc(router.restHandler.AppMetricsEnableDisable).Methods("POST") configRouter.Path("/env/metrics/{appId}/{environmentId}").HandlerFunc(router.restHandler.EnvMetricsEnableDisable).Methods("POST") configRouter.Path("/app-wf"). diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index 29e9c8e826..5148a4279d 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -118,17 +118,14 @@ type AppServiceImpl struct { tokenCache *util3.TokenCache acdAuthConfig *util3.ACDAuthConfig enforcer casbin.Enforcer - enforcerUtil rbac.EnforcerUtil - user user.UserService - appListingRepository repository.AppListingRepository + enforcerUtil rbac.EnforcerUtil + user user.UserService + appListingRepository repository.AppListingRepository appRepository app.AppRepository envRepository repository2.EnvironmentRepository pipelineConfigRepository chartConfig.PipelineConfigRepository configMapRepository chartConfig.ConfigMapRepository chartRepository chartRepoRepository.ChartRepository - appRepo app.AppRepository - appLevelMetricsRepository repository.AppLevelMetricsRepository - envLevelMetricsRepository repository.EnvLevelAppMetricsRepository ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository cdWorkflowRepository pipelineConfig.CdWorkflowRepository commonService commonService.CommonService @@ -211,8 +208,6 @@ func NewAppService( envRepository repository2.EnvironmentRepository, pipelineConfigRepository chartConfig.PipelineConfigRepository, configMapRepository chartConfig.ConfigMapRepository, - appLevelMetricsRepository repository.AppLevelMetricsRepository, - envLevelMetricsRepository repository.EnvLevelAppMetricsRepository, chartRepository chartRepoRepository.ChartRepository, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, @@ -275,8 +270,6 @@ func NewAppService( pipelineConfigRepository: pipelineConfigRepository, configMapRepository: configMapRepository, chartRepository: chartRepository, - appLevelMetricsRepository: appLevelMetricsRepository, - envLevelMetricsRepository: envLevelMetricsRepository, ciPipelineMaterialRepository: ciPipelineMaterialRepository, cdWorkflowRepository: cdWorkflowRepository, commonService: commonService, diff --git a/pkg/chart/ChartService.go b/pkg/chart/ChartService.go index c95a5d05ec..ca6397f113 100644 --- a/pkg/chart/ChartService.go +++ b/pkg/chart/ChartService.go @@ -22,6 +22,8 @@ import ( "context" "encoding/json" "fmt" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/bean" "github.com/devtron-labs/devtron/pkg/resourceQualifiers" "github.com/devtron-labs/devtron/pkg/variables" "github.com/devtron-labs/devtron/pkg/variables/parsers" @@ -78,7 +80,6 @@ type ChartService interface { ChartRefAutocompleteForAppOrEnv(appId int, envId int) (*ChartRefResponse, error) FindPreviousChartByAppId(appId int) (chartTemplate *TemplateRequest, err error) UpgradeForApp(appId int, chartRefId int, newAppOverride map[string]interface{}, userId int32, ctx context.Context) (bool, error) - AppMetricsEnableDisable(appMetricRequest AppMetricEnableDisableRequest) (*AppMetricEnableDisableRequest, error) DeploymentTemplateValidate(ctx context.Context, templatejson interface{}, chartRefId int, scope resourceQualifiers.Scope) (bool, error) JsonSchemaExtractFromFile(chartRefId int) (map[string]interface{}, string, error) GetSchemaAndReadmeForTemplateByChartRefId(chartRefId int) (schema []byte, readme []byte, err error) @@ -115,11 +116,11 @@ type ChartServiceImpl struct { configMapRepository chartConfig.ConfigMapRepository environmentRepository repository4.EnvironmentRepository pipelineRepository pipelineConfig.PipelineRepository - appLevelMetricsRepository repository3.AppLevelMetricsRepository envLevelAppMetricsRepository repository3.EnvLevelAppMetricsRepository client *http.Client deploymentTemplateHistoryService history.DeploymentTemplateHistoryService scopedVariableManager variables.ScopedVariableManager + deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService } func NewChartServiceImpl(chartRepository chartRepoRepository.ChartRepository, @@ -137,12 +138,11 @@ func NewChartServiceImpl(chartRepository chartRepoRepository.ChartRepository, configMapRepository chartConfig.ConfigMapRepository, environmentRepository repository4.EnvironmentRepository, pipelineRepository pipelineConfig.PipelineRepository, - appLevelMetricsRepository repository3.AppLevelMetricsRepository, envLevelAppMetricsRepository repository3.EnvLevelAppMetricsRepository, client *http.Client, deploymentTemplateHistoryService history.DeploymentTemplateHistoryService, scopedVariableManager variables.ScopedVariableManager, -) *ChartServiceImpl { + deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService) *ChartServiceImpl { // cache devtron reference charts list devtronChartList, _ := chartRefRepository.FetchAllChartInfoByUploadFlag(false) @@ -164,11 +164,11 @@ func NewChartServiceImpl(chartRepository chartRepoRepository.ChartRepository, configMapRepository: configMapRepository, environmentRepository: environmentRepository, pipelineRepository: pipelineRepository, - appLevelMetricsRepository: appLevelMetricsRepository, envLevelAppMetricsRepository: envLevelAppMetricsRepository, client: client, deploymentTemplateHistoryService: deploymentTemplateHistoryService, scopedVariableManager: scopedVariableManager, + deployedAppMetricsService: deployedAppMetricsService, } } @@ -443,28 +443,19 @@ func (impl ChartServiceImpl) Create(templateRequest TemplateRequest, ctx context return nil, err } - var appLevelMetrics *repository3.AppLevelMetrics - isAppMetricsSupported, err := impl.CheckIsAppMetricsSupported(templateRequest.ChartRefId) + appLevelMetricsUpdateReq := &bean.DeployedAppMetricsRequest{ + EnableMetrics: templateRequest.IsAppMetricsEnabled, + AppId: templateRequest.AppId, + ChartRefId: templateRequest.ChartRefId, + UserId: templateRequest.UserId, + } + err = impl.deployedAppMetricsService.CheckAndUpdateAppLevelMetrics(ctx, appLevelMetricsUpdateReq) if err != nil { + impl.logger.Errorw("error, CheckAndUpdateAppLevelMetrics", "err", err, "req", appLevelMetricsUpdateReq) return nil, err } - if !(isAppMetricsSupported) { - appMetricsRequest := AppMetricEnableDisableRequest{UserId: templateRequest.UserId, AppId: templateRequest.AppId, IsAppMetricsEnabled: false} - appLevelMetrics, err = impl.updateAppLevelMetrics(&appMetricsRequest) - if err != nil { - impl.logger.Errorw("err while disable app metrics for lower versions", "err", err) - return nil, err - } - } else { - appMetricsRequest := AppMetricEnableDisableRequest{UserId: templateRequest.UserId, AppId: templateRequest.AppId, IsAppMetricsEnabled: templateRequest.IsAppMetricsEnabled} - appLevelMetrics, err = impl.updateAppLevelMetrics(&appMetricsRequest) - if err != nil { - impl.logger.Errorw("err while updating app metrics", "err", err) - return nil, err - } - } - chartVal, err := impl.chartAdaptor(chart, appLevelMetrics) + chartVal, err := impl.chartAdaptor(chart, appLevelMetricsUpdateReq.EnableMetrics) return chartVal, err } @@ -580,19 +571,15 @@ func (impl ChartServiceImpl) CreateChartFromEnvOverride(templateRequest Template return nil, err } - chartVal, err := impl.chartAdaptor(chart, nil) + chartVal, err := impl.chartAdaptor(chart, false) return chartVal, err } // converts db object to bean -func (impl ChartServiceImpl) chartAdaptor(chart *chartRepoRepository.Chart, appLevelMetrics *repository3.AppLevelMetrics) (*TemplateRequest, error) { - var appMetrics bool +func (impl ChartServiceImpl) chartAdaptor(chart *chartRepoRepository.Chart, isAppMetricsEnabled bool) (*TemplateRequest, error) { if chart == nil || chart.Id == 0 { return &TemplateRequest{}, &util.ApiError{UserMessage: "no chart found"} } - if appLevelMetrics != nil { - appMetrics = appLevelMetrics.AppMetrics - } return &TemplateRequest{ RefChartTemplate: chart.ReferenceTemplate, Id: chart.Id, @@ -602,7 +589,7 @@ func (impl ChartServiceImpl) chartAdaptor(chart *chartRepoRepository.Chart, appL RefChartTemplateVersion: impl.getParentChartVersion(chart.ChartVersion), Latest: chart.Latest, ChartRefId: chart.ChartRefId, - IsAppMetricsEnabled: appMetrics, + IsAppMetricsEnabled: isAppMetricsEnabled, IsBasicViewLocked: chart.IsBasicViewLocked, CurrentViewEditor: chart.CurrentViewEditor, }, nil @@ -745,14 +732,12 @@ func (impl ChartServiceImpl) FindLatestChartForAppByAppId(appId int) (chartTempl impl.logger.Errorw("error in fetching chart ", "appId", appId, "err", err) return nil, err } - - appMetrics, err := impl.appLevelMetricsRepository.FindByAppId(appId) - if err != nil && !util.IsErrNoRows(err) { + isAppMetricsEnabled, err := impl.deployedAppMetricsService.GetMetricsFlagByAppIdEvenIfNotInDb(appId) + if err != nil { impl.logger.Errorw("error in fetching app-metrics", "appId", appId, "err", err) return nil, err } - - chartTemplate, err = impl.chartAdaptor(chart, appMetrics) + chartTemplate, err = impl.chartAdaptor(chart, isAppMetricsEnabled) return chartTemplate, err } @@ -762,12 +747,12 @@ func (impl ChartServiceImpl) GetByAppIdAndChartRefId(appId int, chartRefId int) impl.logger.Errorw("error in fetching chart ", "appId", appId, "err", err) return nil, err } - appLevelMetrics, err := impl.appLevelMetricsRepository.FindByAppId(appId) - if err != nil && !util.IsErrNoRows(err) { - impl.logger.Errorw("error in fetching app metrics flag", "err", err) + isAppMetricsEnabled, err := impl.deployedAppMetricsService.GetMetricsFlagByAppIdEvenIfNotInDb(appId) + if err != nil { + impl.logger.Errorw("error in fetching app-metrics", "appId", appId, "err", err) return nil, err } - chartTemplate, err = impl.chartAdaptor(chart, appLevelMetrics) + chartTemplate, err = impl.chartAdaptor(chart, isAppMetricsEnabled) return chartTemplate, err } @@ -855,30 +840,17 @@ func (impl ChartServiceImpl) UpdateAppOverride(ctx context.Context, templateRequ return nil, err } - appMetrics := templateRequest.IsAppMetricsEnabled - isAppMetricsSupported, err := impl.CheckIsAppMetricsSupported(templateRequest.ChartRefId) + appLevelMetricsUpdateReq := &bean.DeployedAppMetricsRequest{ + EnableMetrics: templateRequest.IsAppMetricsEnabled, + AppId: templateRequest.AppId, + ChartRefId: templateRequest.ChartRefId, + UserId: templateRequest.UserId, + } + err = impl.deployedAppMetricsService.CheckAndUpdateAppLevelMetrics(ctx, appLevelMetricsUpdateReq) if err != nil { + impl.logger.Errorw("error, CheckAndUpdateAppLevelMetrics", "err", err, "req", appLevelMetricsUpdateReq) return nil, err } - if appMetrics && !(isAppMetricsSupported) { - appMetricRequest := AppMetricEnableDisableRequest{UserId: templateRequest.UserId, AppId: templateRequest.AppId, IsAppMetricsEnabled: false} - _, span = otel.Tracer("orchestrator").Start(ctx, "updateAppLevelMetrics") - _, err = impl.updateAppLevelMetrics(&appMetricRequest) - span.End() - if err != nil { - impl.logger.Errorw("error in disable app metric flag", "error", err) - return nil, err - } - } else { - appMetricsRequest := AppMetricEnableDisableRequest{UserId: templateRequest.UserId, AppId: templateRequest.AppId, IsAppMetricsEnabled: templateRequest.IsAppMetricsEnabled} - _, span = otel.Tracer("orchestrator").Start(ctx, "updateAppLevelMetrics") - _, err = impl.updateAppLevelMetrics(&appMetricsRequest) - span.End() - if err != nil { - impl.logger.Errorw("err while updating app metrics", "err", err) - return nil, err - } - } _, span = otel.Tracer("orchestrator").Start(ctx, "CreateDeploymentTemplateHistoryFromGlobalTemplate") //creating history entry for deployment template err = impl.deploymentTemplateHistoryService.CreateDeploymentTemplateHistoryFromGlobalTemplate(template, nil, templateRequest.IsAppMetricsEnabled) @@ -921,41 +893,6 @@ func (impl ChartServiceImpl) handleChartTypeChange(currentLatestChart *chartRepo return updatedOverride, nil } -func (impl ChartServiceImpl) updateAppLevelMetrics(appMetricRequest *AppMetricEnableDisableRequest) (*repository3.AppLevelMetrics, error) { - existingAppLevelMetrics, err := impl.appLevelMetricsRepository.FindByAppId(appMetricRequest.AppId) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in app metrics app level flag", "error", err) - return nil, err - } - if existingAppLevelMetrics != nil && existingAppLevelMetrics.Id != 0 { - existingAppLevelMetrics.AppMetrics = appMetricRequest.IsAppMetricsEnabled - err := impl.appLevelMetricsRepository.Update(existingAppLevelMetrics) - if err != nil { - impl.logger.Errorw("failed to update app level metrics flag", "error", err) - return nil, err - } - return existingAppLevelMetrics, nil - } else { - appLevelMetricsNew := &repository3.AppLevelMetrics{ - AppId: appMetricRequest.AppId, - AppMetrics: appMetricRequest.IsAppMetricsEnabled, - InfraMetrics: true, - AuditLog: sql.AuditLog{ - CreatedOn: time.Now(), - UpdatedOn: time.Now(), - CreatedBy: appMetricRequest.UserId, - UpdatedBy: appMetricRequest.UserId, - }, - } - err = impl.appLevelMetricsRepository.Save(appLevelMetricsNew) - if err != nil { - impl.logger.Errorw("error in saving app level metrics flag", "error", err) - return appLevelMetricsNew, err - } - return appLevelMetricsNew, nil - } -} - type IsReady struct { Flag bool `json:"flag"` Message string `json:"message"` @@ -1129,7 +1066,7 @@ func (impl ChartServiceImpl) FindPreviousChartByAppId(appId int) (chartTemplate impl.logger.Errorw("error in fetching chart ", "appId", appId, "err", err) return nil, err } - chartTemplate, err = impl.chartAdaptor(chart, nil) + chartTemplate, err = impl.chartAdaptor(chart, false) return chartTemplate, err } @@ -1207,13 +1144,12 @@ func (impl ChartServiceImpl) UpgradeForApp(appId int, chartRefId int, newAppOver impl.logger.Errorw("error in getting env level app metrics", "err", err, "appId", appId, "envId", envOverrideNew.TargetEnvironment) return false, err } else if err == pg.ErrNoRows { - appLevelAppMetrics, err := impl.appLevelMetricsRepository.FindByAppId(appId) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in getting app level app metrics", "err", err, "appId", appId) + isAppLevelMetricsEnabled, err := impl.deployedAppMetricsService.GetMetricsFlagByAppIdEvenIfNotInDb(appId) + if err != nil { + impl.logger.Errorw("error, GetMetricsFlagByAppIdEvenIfNotInDb", "err", err, "appId", appId) return false, err - } else if err == nil { - isAppMetricsEnabled = appLevelAppMetrics.AppMetrics } + isAppMetricsEnabled = isAppLevelMetricsEnabled } else { isAppMetricsEnabled = *envLevelAppMetrics.AppMetrics } @@ -1232,65 +1168,6 @@ func (impl ChartServiceImpl) UpgradeForApp(appId int, chartRefId int, newAppOver return true, nil } -// below method is deprecated - -func (impl ChartServiceImpl) AppMetricsEnableDisable(appMetricRequest AppMetricEnableDisableRequest) (*AppMetricEnableDisableRequest, error) { - currentChart, err := impl.chartRepository.FindLatestChartForAppByAppId(appMetricRequest.AppId) - if err != nil && pg.ErrNoRows != err { - impl.logger.Error(err) - return nil, err - } - if pg.ErrNoRows == err { - impl.logger.Errorw("no chart configured for this app", "appId", appMetricRequest.AppId) - err = &util.ApiError{ - HttpStatusCode: http.StatusNotFound, - InternalMessage: "no chart configured for this app", - UserMessage: "no chart configured for this app", - } - return nil, err - } - // validate app metrics compatibility - refChart, err := impl.chartRefRepository.FindById(currentChart.ChartRefId) - if err != nil { - impl.logger.Error(err) - return nil, err - } - if appMetricRequest.IsAppMetricsEnabled == true { - chartMajorVersion, chartMinorVersion, err := util2.ExtractChartVersion(currentChart.ChartVersion) - if err != nil { - impl.logger.Errorw("chart version parsing", "err", err) - return nil, err - } - - if !refChart.UserUploaded && !(chartMajorVersion >= 3 && chartMinorVersion >= 7) { - err = &util.ApiError{ - InternalMessage: "chart version in not compatible for app metrics", - UserMessage: "chart version in not compatible for app metrics", - } - return nil, err - } - } - //update or create app level app metrics - appLevelMetrics, err := impl.updateAppLevelMetrics(&appMetricRequest) - if err != nil { - impl.logger.Errorw("error in saving app level metrics flag", "error", err) - return nil, err - } - //updating audit log details of chart as history service uses it - currentChart.UpdatedOn = time.Now() - currentChart.UpdatedBy = appMetricRequest.UserId - //creating history entry for deployment template - err = impl.deploymentTemplateHistoryService.CreateDeploymentTemplateHistoryFromGlobalTemplate(currentChart, nil, appMetricRequest.IsAppMetricsEnabled) - if err != nil { - impl.logger.Errorw("error in creating entry for deployment template history", "err", err, "chart", currentChart) - return nil, err - } - if appLevelMetrics.Id > 0 { - return &appMetricRequest, nil - } - return nil, err -} - const memoryPattern = `"1000Mi" or "1Gi"` const cpuPattern = `"50m" or "0.05"` const cpu = "cpu" diff --git a/pkg/deployment/manifest/deployedAppMetrics/DeployedAppMetrics.go b/pkg/deployment/manifest/deployedAppMetrics/DeployedAppMetrics.go new file mode 100644 index 0000000000..2dcf0096d3 --- /dev/null +++ b/pkg/deployment/manifest/deployedAppMetrics/DeployedAppMetrics.go @@ -0,0 +1,117 @@ +package deployedAppMetrics + +import ( + "context" + interalRepo "github.com/devtron-labs/devtron/internal/sql/repository" + "github.com/devtron-labs/devtron/internal/util" + chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/bean" + "github.com/devtron-labs/devtron/pkg/sql" + "github.com/go-pg/pg" + "go.opentelemetry.io/otel" + "go.uber.org/zap" + "time" +) + +type DeployedAppMetricsService interface { + GetMetricsFlagByAppIdEvenIfNotInDb(appId int) (bool, error) + CheckAndUpdateAppLevelMetrics(ctx context.Context, req *bean.DeployedAppMetricsRequest) error +} + +type DeployedAppMetricsServiceImpl struct { + logger *zap.SugaredLogger + chartRefRepository chartRepoRepository.ChartRefRepository + appLevelMetricsRepository interalRepo.AppLevelMetricsRepository + envLevelMetricsRepository interalRepo.EnvLevelAppMetricsRepository +} + +func NewDeployedAppMetricsServiceImpl(logger *zap.SugaredLogger, + chartRefRepository chartRepoRepository.ChartRefRepository, + appLevelMetricsRepository interalRepo.AppLevelMetricsRepository, + envLevelMetricsRepository interalRepo.EnvLevelAppMetricsRepository) *DeployedAppMetricsServiceImpl { + return &DeployedAppMetricsServiceImpl{ + logger: logger, + chartRefRepository: chartRefRepository, + appLevelMetricsRepository: appLevelMetricsRepository, + envLevelMetricsRepository: envLevelMetricsRepository, + } +} + +func (impl *DeployedAppMetricsServiceImpl) GetMetricsFlagByAppIdEvenIfNotInDb(appId int) (bool, error) { + isAppMetricsEnabled := false + appMetrics, err := impl.appLevelMetricsRepository.FindByAppId(appId) + if err != nil && !util.IsErrNoRows(err) { + impl.logger.Errorw("error in fetching app level metrics", "appId", appId, "err", err) + return isAppMetricsEnabled, err + } + if appMetrics != nil { + isAppMetricsEnabled = appMetrics.AppMetrics + } + return isAppMetricsEnabled, nil +} + +// CheckAndUpdateAppLevelMetrics - this method checks whether chart being used supports metrics or not and update accordingly +func (impl *DeployedAppMetricsServiceImpl) CheckAndUpdateAppLevelMetrics(ctx context.Context, req *bean.DeployedAppMetricsRequest) error { + isAppMetricsSupported, err := impl.checkIsAppMetricsSupported(req.ChartRefId) + if err != nil { + return err + } + if !(isAppMetricsSupported) { + //chart does not have metrics support, disabling + req.EnableMetrics = false + } + _, span := otel.Tracer("orchestrator").Start(ctx, "updateAppLevelMetrics") + _, err = impl.updateAppLevelMetrics(req) + span.End() + if err != nil { + impl.logger.Errorw("error in disable app metric flag", "error", err) + return err + } + return nil +} + +func (impl *DeployedAppMetricsServiceImpl) checkIsAppMetricsSupported(chartRefId int) (bool, error) { + chartRefValue, err := impl.chartRefRepository.FindById(chartRefId) + if err != nil { + impl.logger.Errorw("error in finding reference chart by id", "err", err) + return false, nil + } + return chartRefValue.IsAppMetricsSupported, nil +} + +func (impl *DeployedAppMetricsServiceImpl) updateAppLevelMetrics(req *bean.DeployedAppMetricsRequest) (*interalRepo.AppLevelMetrics, error) { + existingAppLevelMetrics, err := impl.appLevelMetricsRepository.FindByAppId(req.AppId) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in app metrics app level flag", "error", err) + return nil, err + } + if existingAppLevelMetrics != nil && existingAppLevelMetrics.Id != 0 { + existingAppLevelMetrics.AppMetrics = req.EnableMetrics + existingAppLevelMetrics.UpdatedBy = req.UserId + existingAppLevelMetrics.UpdatedOn = time.Now() + err := impl.appLevelMetricsRepository.Update(existingAppLevelMetrics) + if err != nil { + impl.logger.Errorw("error in to updating app level metrics", "error", err, "model", existingAppLevelMetrics) + return nil, err + } + return existingAppLevelMetrics, nil + } else { + appLevelMetricsNew := &interalRepo.AppLevelMetrics{ + AppId: req.AppId, + AppMetrics: req.EnableMetrics, + InfraMetrics: true, + AuditLog: sql.AuditLog{ + CreatedOn: time.Now(), + UpdatedOn: time.Now(), + CreatedBy: req.UserId, + UpdatedBy: req.UserId, + }, + } + err = impl.appLevelMetricsRepository.Save(appLevelMetricsNew) + if err != nil { + impl.logger.Errorw("error in saving app level metrics flag", "error", err, "model", appLevelMetricsNew) + return appLevelMetricsNew, err + } + return appLevelMetricsNew, nil + } +} diff --git a/pkg/deployment/manifest/deployedAppMetrics/bean/bean.go b/pkg/deployment/manifest/deployedAppMetrics/bean/bean.go new file mode 100644 index 0000000000..42761fdc48 --- /dev/null +++ b/pkg/deployment/manifest/deployedAppMetrics/bean/bean.go @@ -0,0 +1,9 @@ +package bean + +type DeployedAppMetricsRequest struct { + EnableMetrics bool + AppId int + EnvId int // if not zero then request for override + ChartRefId int + UserId int32 +} diff --git a/pkg/deployment/manifest/deploymentTemplate/DeploymentTemplateService.go b/pkg/deployment/manifest/deploymentTemplate/DeploymentTemplateService.go new file mode 100644 index 0000000000..2c43ce535a --- /dev/null +++ b/pkg/deployment/manifest/deploymentTemplate/DeploymentTemplateService.go @@ -0,0 +1,16 @@ +package deploymentTemplate + +import "go.uber.org/zap" + +type DeploymentTemplateService interface { +} + +type DeploymentTemplateServiceImpl struct { + logger *zap.SugaredLogger +} + +func NewDeploymentTemplateServiceImpl(logger *zap.SugaredLogger) *DeploymentTemplateServiceImpl { + return &DeploymentTemplateServiceImpl{ + logger: logger, + } +} diff --git a/pkg/pipeline/DeploymentConfigService.go b/pkg/pipeline/DeploymentConfigService.go index fa7a7a8972..673bc39365 100644 --- a/pkg/pipeline/DeploymentConfigService.go +++ b/pkg/pipeline/DeploymentConfigService.go @@ -8,6 +8,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/pkg/bean" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" "github.com/devtron-labs/devtron/pkg/pipeline/history" repository2 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" "github.com/devtron-labs/devtron/pkg/resourceQualifiers" @@ -29,12 +30,12 @@ type DeploymentConfigServiceImpl struct { chartRepository chartRepoRepository.ChartRepository pipelineRepository pipelineConfig.PipelineRepository envLevelAppMetricsRepository repository.EnvLevelAppMetricsRepository - appLevelMetricsRepository repository.AppLevelMetricsRepository pipelineConfigRepository chartConfig.PipelineConfigRepository configMapRepository chartConfig.ConfigMapRepository configMapHistoryService history.ConfigMapHistoryService chartRefRepository chartRepoRepository.ChartRefRepository scopedVariableManager variables.ScopedVariableCMCSManager + deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService } func NewDeploymentConfigServiceImpl(logger *zap.SugaredLogger, @@ -42,25 +43,24 @@ func NewDeploymentConfigServiceImpl(logger *zap.SugaredLogger, chartRepository chartRepoRepository.ChartRepository, pipelineRepository pipelineConfig.PipelineRepository, envLevelAppMetricsRepository repository.EnvLevelAppMetricsRepository, - appLevelMetricsRepository repository.AppLevelMetricsRepository, pipelineConfigRepository chartConfig.PipelineConfigRepository, configMapRepository chartConfig.ConfigMapRepository, configMapHistoryService history.ConfigMapHistoryService, chartRefRepository chartRepoRepository.ChartRefRepository, scopedVariableManager variables.ScopedVariableCMCSManager, -) *DeploymentConfigServiceImpl { + deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService) *DeploymentConfigServiceImpl { return &DeploymentConfigServiceImpl{ logger: logger, envConfigOverrideRepository: envConfigOverrideRepository, chartRepository: chartRepository, pipelineRepository: pipelineRepository, envLevelAppMetricsRepository: envLevelAppMetricsRepository, - appLevelMetricsRepository: appLevelMetricsRepository, pipelineConfigRepository: pipelineConfigRepository, configMapRepository: configMapRepository, configMapHistoryService: configMapHistoryService, chartRefRepository: chartRefRepository, scopedVariableManager: scopedVariableManager, + deployedAppMetricsService: deployedAppMetricsService, } } @@ -102,12 +102,12 @@ func (impl *DeploymentConfigServiceImpl) GetLatestDeploymentTemplateConfig(ctx c if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in getting env level app metrics", "err", err, "appId", pipeline.AppId, "envId", pipeline.EnvironmentId) } else if err == pg.ErrNoRows { - appLevelAppMetrics, err := impl.appLevelMetricsRepository.FindByAppId(pipeline.AppId) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in getting app level app metrics", "err", err, "appId", pipeline.AppId) - } else if err == nil { - isAppMetricsEnabled = appLevelAppMetrics.AppMetrics + isAppLevelMetricsEnabled, err := impl.deployedAppMetricsService.GetMetricsFlagByAppIdEvenIfNotInDb(pipeline.AppId) + if err != nil { + impl.logger.Errorw("error, GetMetricsFlagByAppIdEvenIfNotInDb", "err", err, "appId", pipeline.AppId) + return nil, err } + isAppMetricsEnabled = isAppLevelMetricsEnabled } else { isAppMetricsEnabled = *envLevelAppMetrics.AppMetrics } diff --git a/pkg/pipeline/DeploymentPipelineConfigService.go b/pkg/pipeline/DeploymentPipelineConfigService.go index ca0c9df270..8a80c10bd3 100644 --- a/pkg/pipeline/DeploymentPipelineConfigService.go +++ b/pkg/pipeline/DeploymentPipelineConfigService.go @@ -41,6 +41,7 @@ import ( chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" "github.com/devtron-labs/devtron/pkg/cluster" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/history" repository4 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" @@ -135,7 +136,6 @@ type CdPipelineConfigServiceImpl struct { resourceGroupService resourceGroup2.ResourceGroupService chartTemplateService util.ChartTemplateService propertiesConfigService PropertiesConfigService - appLevelMetricsRepository repository.AppLevelMetricsRepository deploymentTemplateHistoryService history.DeploymentTemplateHistoryService scopedVariableManager variables.ScopedVariableManager deploymentConfig *DeploymentServiceTypeConfig @@ -146,6 +146,7 @@ type CdPipelineConfigServiceImpl struct { ciPipelineConfigService CiPipelineConfigService buildPipelineSwitchService BuildPipelineSwitchService argoClientWrapperService argocdServer.ArgoClientWrapperService + deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService } func NewCdPipelineConfigServiceImpl(logger *zap.SugaredLogger, pipelineRepository pipelineConfig.PipelineRepository, @@ -159,12 +160,13 @@ func NewCdPipelineConfigServiceImpl(logger *zap.SugaredLogger, pipelineRepositor pipelineStrategyHistoryService history.PipelineStrategyHistoryService, chartRepository chartRepoRepository.ChartRepository, resourceGroupService resourceGroup2.ResourceGroupService, chartTemplateService util.ChartTemplateService, propertiesConfigService PropertiesConfigService, - appLevelMetricsRepository repository.AppLevelMetricsRepository, deploymentTemplateHistoryService history.DeploymentTemplateHistoryService, + deploymentTemplateHistoryService history.DeploymentTemplateHistoryService, scopedVariableManager variables.ScopedVariableManager, deploymentConfig *DeploymentServiceTypeConfig, application application.ServiceClient, customTagService CustomTagService, pipelineConfigListenerService PipelineConfigListenerService, devtronAppCMCSService DevtronAppCMCSService, ciPipelineConfigService CiPipelineConfigService, buildPipelineSwitchService BuildPipelineSwitchService, - argoClientWrapperService argocdServer.ArgoClientWrapperService) *CdPipelineConfigServiceImpl { + argoClientWrapperService argocdServer.ArgoClientWrapperService, + deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService) *CdPipelineConfigServiceImpl { return &CdPipelineConfigServiceImpl{ logger: logger, pipelineRepository: pipelineRepository, @@ -188,7 +190,6 @@ func NewCdPipelineConfigServiceImpl(logger *zap.SugaredLogger, pipelineRepositor resourceGroupService: resourceGroupService, chartTemplateService: chartTemplateService, propertiesConfigService: propertiesConfigService, - appLevelMetricsRepository: appLevelMetricsRepository, deploymentTemplateHistoryService: deploymentTemplateHistoryService, scopedVariableManager: scopedVariableManager, deploymentConfig: deploymentConfig, @@ -199,6 +200,7 @@ func NewCdPipelineConfigServiceImpl(logger *zap.SugaredLogger, pipelineRepositor ciPipelineConfigService: ciPipelineConfigService, buildPipelineSwitchService: buildPipelineSwitchService, argoClientWrapperService: argoClientWrapperService, + deployedAppMetricsService: deployedAppMetricsService, } } @@ -1733,12 +1735,12 @@ func (impl *CdPipelineConfigServiceImpl) createCdPipeline(ctx context.Context, a } //getting global app metrics for cd pipeline create because env level metrics is not created yet appLevelAppMetricsEnabled := false - appLevelMetrics, err := impl.appLevelMetricsRepository.FindByAppId(app.Id) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in getting app level metrics app level", "error", err) - } else if err == nil { - appLevelAppMetricsEnabled = appLevelMetrics.AppMetrics + isAppLevelMetricsEnabled, err := impl.deployedAppMetricsService.GetMetricsFlagByAppIdEvenIfNotInDb(app.Id) + if err != nil { + impl.logger.Errorw("error, GetMetricsFlagByAppIdEvenIfNotInDb", "err", err, "appId", app.Id) + return 0, err } + appLevelAppMetricsEnabled = isAppLevelMetricsEnabled err = impl.deploymentTemplateHistoryService.CreateDeploymentTemplateHistoryFromEnvOverrideTemplate(envOverride, tx, appLevelAppMetricsEnabled, pipelineId) if err != nil { impl.logger.Errorw("error in creating entry for env deployment template history", "err", err, "envOverride", envOverride) diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index 342bbab976..1391bc5ce6 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -22,6 +22,7 @@ import ( "encoding/json" errors3 "errors" "fmt" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" "path" "strconv" "strings" @@ -188,7 +189,6 @@ type WorkflowDagExecutorImpl struct { helmAppClient client2.HelmAppClient chartRefRepository chartRepoRepository.ChartRefRepository environmentConfigRepository chartConfig.EnvConfigOverrideRepository - appLevelMetricsRepository repository.AppLevelMetricsRepository envLevelMetricsRepository repository.EnvLevelAppMetricsRepository dbMigrationConfigRepository pipelineConfig.DbMigrationConfigRepository mergeUtil *util.MergeUtil @@ -199,6 +199,7 @@ type WorkflowDagExecutorImpl struct { pipelineConfigListenerService PipelineConfigListenerService customTagService CustomTagService ACDConfig *argocdServer.ACDConfig + deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService } const kedaAutoscaling = "kedaAutoscaling" @@ -296,7 +297,6 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi helmAppClient client2.HelmAppClient, chartRefRepository chartRepoRepository.ChartRefRepository, environmentConfigRepository chartConfig.EnvConfigOverrideRepository, - appLevelMetricsRepository repository.AppLevelMetricsRepository, envLevelMetricsRepository repository.EnvLevelAppMetricsRepository, dbMigrationConfigRepository pipelineConfig.DbMigrationConfigRepository, mergeUtil *util.MergeUtil, @@ -307,7 +307,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi pipelineConfigListenerService PipelineConfigListenerService, customTagService CustomTagService, ACDConfig *argocdServer.ACDConfig, -) *WorkflowDagExecutorImpl { + deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService) *WorkflowDagExecutorImpl { wde := &WorkflowDagExecutorImpl{logger: Logger, pipelineRepository: pipelineRepository, cdWorkflowRepository: cdWorkflowRepository, @@ -373,7 +373,6 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi helmAppClient: helmAppClient, chartRefRepository: chartRefRepository, environmentConfigRepository: environmentConfigRepository, - appLevelMetricsRepository: appLevelMetricsRepository, envLevelMetricsRepository: envLevelMetricsRepository, dbMigrationConfigRepository: dbMigrationConfigRepository, mergeUtil: mergeUtil, @@ -384,6 +383,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi pipelineConfigListenerService: pipelineConfigListenerService, customTagService: customTagService, ACDConfig: ACDConfig, + deployedAppMetricsService: deployedAppMetricsService, } config, err := types.GetCdConfig() if err != nil { @@ -3810,14 +3810,12 @@ func (impl *WorkflowDagExecutorImpl) GetAppMetricsByTriggerType(overrideRequest } else if overrideRequest.DeploymentWithConfig == bean.DEPLOYMENT_CONFIG_TYPE_LAST_SAVED { _, span := otel.Tracer("orchestrator").Start(ctx, "appLevelMetricsRepository.FindByAppId") - appLevelMetrics, err := impl.appLevelMetricsRepository.FindByAppId(overrideRequest.AppId) - span.End() - if err != nil && !util.IsErrNoRows(err) { - impl.logger.Errorw("err", err) - return appMetrics, &util.ApiError{InternalMessage: "unable to fetch app level metrics flag"} + isAppLevelMetricsEnabled, err := impl.deployedAppMetricsService.GetMetricsFlagByAppIdEvenIfNotInDb(overrideRequest.AppId) + if err != nil { + impl.logger.Errorw("error, GetMetricsFlagByAppIdEvenIfNotInDb", "err", err, "appId", overrideRequest.AppId) + return appMetrics, err } - appMetrics = appLevelMetrics.AppMetrics - + appMetrics = isAppLevelMetricsEnabled _, span = otel.Tracer("orchestrator").Start(ctx, "envLevelMetricsRepository.FindByAppIdAndEnvId") envLevelMetrics, err := impl.envLevelMetricsRepository.FindByAppIdAndEnvId(overrideRequest.AppId, overrideRequest.EnvId) span.End() diff --git a/pkg/pipeline/history/DeploymentTemplateHistoryService.go b/pkg/pipeline/history/DeploymentTemplateHistoryService.go index 645d42c1cf..1d41cf373d 100644 --- a/pkg/pipeline/history/DeploymentTemplateHistoryService.go +++ b/pkg/pipeline/history/DeploymentTemplateHistoryService.go @@ -2,6 +2,7 @@ package history import ( "context" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" "time" repository2 "github.com/devtron-labs/devtron/internal/sql/repository" @@ -40,10 +41,10 @@ type DeploymentTemplateHistoryServiceImpl struct { chartRepository chartRepoRepository.ChartRepository chartRefRepository chartRepoRepository.ChartRefRepository envLevelAppMetricsRepository repository2.EnvLevelAppMetricsRepository - appLevelMetricsRepository repository2.AppLevelMetricsRepository userService user.UserService cdWorkflowRepository pipelineConfig.CdWorkflowRepository scopedVariableManager variables.ScopedVariableManager + deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService } func NewDeploymentTemplateHistoryServiceImpl(logger *zap.SugaredLogger, deploymentTemplateHistoryRepository repository.DeploymentTemplateHistoryRepository, @@ -51,11 +52,10 @@ func NewDeploymentTemplateHistoryServiceImpl(logger *zap.SugaredLogger, deployme chartRepository chartRepoRepository.ChartRepository, chartRefRepository chartRepoRepository.ChartRefRepository, envLevelAppMetricsRepository repository2.EnvLevelAppMetricsRepository, - appLevelMetricsRepository repository2.AppLevelMetricsRepository, userService user.UserService, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, scopedVariableManager variables.ScopedVariableManager, -) *DeploymentTemplateHistoryServiceImpl { + deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService) *DeploymentTemplateHistoryServiceImpl { return &DeploymentTemplateHistoryServiceImpl{ logger: logger, deploymentTemplateHistoryRepository: deploymentTemplateHistoryRepository, @@ -63,10 +63,10 @@ func NewDeploymentTemplateHistoryServiceImpl(logger *zap.SugaredLogger, deployme chartRepository: chartRepository, chartRefRepository: chartRefRepository, envLevelAppMetricsRepository: envLevelAppMetricsRepository, - appLevelMetricsRepository: appLevelMetricsRepository, userService: userService, cdWorkflowRepository: cdWorkflowRepository, scopedVariableManager: scopedVariableManager, + deployedAppMetricsService: deployedAppMetricsService, } } @@ -214,13 +214,12 @@ func (impl DeploymentTemplateHistoryServiceImpl) CreateDeploymentTemplateHistory impl.logger.Errorw("error in getting env level app metrics", "err", err, "appId", pipeline.AppId, "envId", pipeline.EnvironmentId) return nil, err } else if err == pg.ErrNoRows { - appLevelAppMetrics, err := impl.appLevelMetricsRepository.FindByAppId(pipeline.AppId) - if err != nil && err != pg.ErrNoRows { + isAppLevelMetricsEnabled, err := impl.deployedAppMetricsService.GetMetricsFlagByAppIdEvenIfNotInDb(pipeline.AppId) + if err != nil { impl.logger.Errorw("error in getting app level app metrics", "err", err, "appId", pipeline.AppId) return nil, err - } else if err == nil { - isAppMetricsEnabled = appLevelAppMetrics.AppMetrics } + isAppMetricsEnabled = isAppLevelMetricsEnabled } else { isAppMetricsEnabled = *envLevelAppMetrics.AppMetrics } diff --git a/wire_gen.go b/wire_gen.go index 600ede7536..8520b05196 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -102,6 +102,7 @@ import ( "github.com/devtron-labs/devtron/pkg/clusterTerminalAccess" "github.com/devtron-labs/devtron/pkg/commonService" delete2 "github.com/devtron-labs/devtron/pkg/delete" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" "github.com/devtron-labs/devtron/pkg/deploymentGroup" "github.com/devtron-labs/devtron/pkg/devtronResource" repository8 "github.com/devtron-labs/devtron/pkg/devtronResource/repository" @@ -305,8 +306,6 @@ func InitializeApp() (*App, error) { appListingRepositoryImpl := repository.NewAppListingRepositoryImpl(sugaredLogger, db, appListingRepositoryQueryBuilder, environmentRepositoryImpl) pipelineConfigRepositoryImpl := chartConfig.NewPipelineConfigRepository(db) configMapRepositoryImpl := chartConfig.NewConfigMapRepositoryImpl(sugaredLogger, db) - appLevelMetricsRepositoryImpl := repository.NewAppLevelMetricsRepositoryImpl(db, sugaredLogger) - envLevelAppMetricsRepositoryImpl := repository.NewEnvLevelAppMetricsRepositoryImpl(db, sugaredLogger) chartRepositoryImpl := chartRepoRepository.NewChartRepository(db) dockerArtifactStoreRepositoryImpl := repository5.NewDockerArtifactStoreRepositoryImpl(db) gitProviderRepositoryImpl := repository.NewGitProviderRepositoryImpl(db) @@ -355,11 +354,14 @@ func InitializeApp() (*App, error) { configMapHistoryServiceImpl := history.NewConfigMapHistoryServiceImpl(sugaredLogger, configMapHistoryRepositoryImpl, pipelineRepositoryImpl, configMapRepositoryImpl, userServiceImpl, scopedVariableCMCSManagerImpl) deploymentTemplateHistoryRepositoryImpl := repository6.NewDeploymentTemplateHistoryRepositoryImpl(sugaredLogger, db) chartRefRepositoryImpl := chartRepoRepository.NewChartRefRepositoryImpl(db) + envLevelAppMetricsRepositoryImpl := repository.NewEnvLevelAppMetricsRepositoryImpl(db, sugaredLogger) scopedVariableManagerImpl, err := variables.NewScopedVariableManagerImpl(sugaredLogger, scopedVariableServiceImpl, variableEntityMappingServiceImpl, variableSnapshotHistoryServiceImpl, variableTemplateParserImpl) if err != nil { return nil, err } - deploymentTemplateHistoryServiceImpl := history.NewDeploymentTemplateHistoryServiceImpl(sugaredLogger, deploymentTemplateHistoryRepositoryImpl, pipelineRepositoryImpl, chartRepositoryImpl, chartRefRepositoryImpl, envLevelAppMetricsRepositoryImpl, appLevelMetricsRepositoryImpl, userServiceImpl, cdWorkflowRepositoryImpl, scopedVariableManagerImpl) + appLevelMetricsRepositoryImpl := repository.NewAppLevelMetricsRepositoryImpl(db, sugaredLogger) + deployedAppMetricsServiceImpl := deployedAppMetrics.NewDeployedAppMetricsServiceImpl(sugaredLogger, chartRefRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl) + deploymentTemplateHistoryServiceImpl := history.NewDeploymentTemplateHistoryServiceImpl(sugaredLogger, deploymentTemplateHistoryRepositoryImpl, pipelineRepositoryImpl, chartRepositoryImpl, chartRefRepositoryImpl, envLevelAppMetricsRepositoryImpl, userServiceImpl, cdWorkflowRepositoryImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl) chartWorkingDir := _wireChartWorkingDirValue globalEnvVariables, err := util3.GetGlobalEnvVariables() if err != nil { @@ -373,7 +375,7 @@ func InitializeApp() (*App, error) { Logger: sugaredLogger, } repositoryServiceClientImpl := repository9.NewServiceClientImpl(sugaredLogger, argoCDConnectionManagerImpl) - chartServiceImpl := chart.NewChartServiceImpl(chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, appRepositoryImpl, refChartDir, defaultChart, utilMergeUtil, repositoryServiceClientImpl, chartRefRepositoryImpl, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, httpClient, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl) + chartServiceImpl := chart.NewChartServiceImpl(chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, appRepositoryImpl, refChartDir, defaultChart, utilMergeUtil, repositoryServiceClientImpl, chartRefRepositoryImpl, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, envLevelAppMetricsRepositoryImpl, httpClient, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl) devtronSecretConfig, err := util3.GetDevtronSecretName() if err != nil { return nil, err @@ -426,7 +428,7 @@ func InitializeApp() (*App, error) { k8sCommonServiceImpl := k8s2.NewK8sCommonServiceImpl(sugaredLogger, k8sUtil, clusterServiceImplExtended) manifestPushConfigRepositoryImpl := repository11.NewManifestPushConfigRepository(sugaredLogger, db) gitOpsManifestPushServiceImpl := app2.NewGitOpsManifestPushServiceImpl(sugaredLogger, chartTemplateServiceImpl, chartServiceImpl, gitOpsConfigRepositoryImpl, gitFactory, pipelineStatusTimelineServiceImpl, pipelineStatusTimelineRepositoryImpl, acdConfig) - appServiceImpl := app2.NewAppService(envConfigOverrideRepositoryImpl, pipelineOverrideRepositoryImpl, mergeUtil, sugaredLogger, ciArtifactRepositoryImpl, pipelineRepositoryImpl, dbMigrationConfigRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, applicationServiceClientImpl, tokenCache, acdAuthConfig, enforcerImpl, enforcerUtilImpl, userServiceImpl, appListingRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, chartRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, imageScanDeployInfoRepositoryImpl, imageScanHistoryRepositoryImpl, argoK8sClientImpl, gitFactory, pipelineStrategyHistoryServiceImpl, configMapHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, chartTemplateServiceImpl, refChartDir, chartRefRepositoryImpl, chartServiceImpl, helmAppClientImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, appCrudOperationServiceImpl, configMapHistoryRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, dockerRegistryIpsConfigServiceImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, gitOpsConfigRepositoryImpl, appStatusServiceImpl, installedAppRepositoryImpl, appStoreDeploymentServiceImpl, k8sCommonServiceImpl, installedAppVersionHistoryRepositoryImpl, globalEnvVariables, helmAppServiceImpl, manifestPushConfigRepositoryImpl, gitOpsManifestPushServiceImpl, argoClientWrapperServiceImpl, scopedVariableCMCSManagerImpl, acdConfig) + appServiceImpl := app2.NewAppService(envConfigOverrideRepositoryImpl, pipelineOverrideRepositoryImpl, mergeUtil, sugaredLogger, ciArtifactRepositoryImpl, pipelineRepositoryImpl, dbMigrationConfigRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, applicationServiceClientImpl, tokenCache, acdAuthConfig, enforcerImpl, enforcerUtilImpl, userServiceImpl, appListingRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, imageScanDeployInfoRepositoryImpl, imageScanHistoryRepositoryImpl, argoK8sClientImpl, gitFactory, pipelineStrategyHistoryServiceImpl, configMapHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, chartTemplateServiceImpl, refChartDir, chartRefRepositoryImpl, chartServiceImpl, helmAppClientImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, appCrudOperationServiceImpl, configMapHistoryRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, dockerRegistryIpsConfigServiceImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, gitOpsConfigRepositoryImpl, appStatusServiceImpl, installedAppRepositoryImpl, appStoreDeploymentServiceImpl, k8sCommonServiceImpl, installedAppVersionHistoryRepositoryImpl, globalEnvVariables, helmAppServiceImpl, manifestPushConfigRepositoryImpl, gitOpsManifestPushServiceImpl, argoClientWrapperServiceImpl, scopedVariableCMCSManagerImpl, acdConfig) validate, err := util.IntValidator() if err != nil { return nil, err @@ -467,10 +469,10 @@ func InitializeApp() (*App, error) { customTagServiceImpl := pipeline.NewCustomTagService(sugaredLogger, imageTagRepositoryImpl) pluginInputVariableParserImpl := pipeline.NewPluginInputVariableParserImpl(sugaredLogger, dockerRegistryConfigImpl, customTagServiceImpl) pipelineConfigListenerServiceImpl := pipeline.NewPipelineConfigListenerServiceImpl(sugaredLogger) - workflowDagExecutorImpl := pipeline.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, pubSubClientServiceImpl, appServiceImpl, workflowServiceImpl, ciArtifactRepositoryImpl, ciPipelineRepositoryImpl, materialRepositoryImpl, pipelineOverrideRepositoryImpl, userServiceImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, enforcerImpl, enforcerUtilImpl, tokenCache, acdAuthConfig, eventSimpleFactoryImpl, eventRESTClientImpl, cvePolicyRepositoryImpl, imageScanResultRepositoryImpl, appWorkflowRepositoryImpl, prePostCdScriptHistoryServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineServiceImpl, ciTemplateRepositoryImpl, ciWorkflowRepositoryImpl, appLabelRepositoryImpl, clientImpl, pipelineStageServiceImpl, k8sCommonServiceImpl, variableSnapshotHistoryServiceImpl, globalPluginServiceImpl, pluginInputVariableParserImpl, scopedVariableCMCSManagerImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineStrategyHistoryServiceImpl, manifestPushConfigRepositoryImpl, gitOpsManifestPushServiceImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, appCrudOperationServiceImpl, pipelineConfigRepositoryImpl, dockerRegistryIpsConfigServiceImpl, chartRepositoryImpl, chartTemplateServiceImpl, pipelineStrategyHistoryRepositoryImpl, appRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, argoK8sClientImpl, configMapRepositoryImpl, configMapHistoryRepositoryImpl, refChartDir, helmAppServiceImpl, helmAppClientImpl, chartRefRepositoryImpl, envConfigOverrideRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, dbMigrationConfigRepositoryImpl, mergeUtil, gitOpsConfigRepositoryImpl, gitFactory, applicationServiceClientImpl, argoClientWrapperServiceImpl, pipelineConfigListenerServiceImpl, customTagServiceImpl, acdConfig) + workflowDagExecutorImpl := pipeline.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, pubSubClientServiceImpl, appServiceImpl, workflowServiceImpl, ciArtifactRepositoryImpl, ciPipelineRepositoryImpl, materialRepositoryImpl, pipelineOverrideRepositoryImpl, userServiceImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, enforcerImpl, enforcerUtilImpl, tokenCache, acdAuthConfig, eventSimpleFactoryImpl, eventRESTClientImpl, cvePolicyRepositoryImpl, imageScanResultRepositoryImpl, appWorkflowRepositoryImpl, prePostCdScriptHistoryServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineServiceImpl, ciTemplateRepositoryImpl, ciWorkflowRepositoryImpl, appLabelRepositoryImpl, clientImpl, pipelineStageServiceImpl, k8sCommonServiceImpl, variableSnapshotHistoryServiceImpl, globalPluginServiceImpl, pluginInputVariableParserImpl, scopedVariableCMCSManagerImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineStrategyHistoryServiceImpl, manifestPushConfigRepositoryImpl, gitOpsManifestPushServiceImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, appCrudOperationServiceImpl, pipelineConfigRepositoryImpl, dockerRegistryIpsConfigServiceImpl, chartRepositoryImpl, chartTemplateServiceImpl, pipelineStrategyHistoryRepositoryImpl, appRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, argoK8sClientImpl, configMapRepositoryImpl, configMapHistoryRepositoryImpl, refChartDir, helmAppServiceImpl, helmAppClientImpl, chartRefRepositoryImpl, envConfigOverrideRepositoryImpl, envLevelAppMetricsRepositoryImpl, dbMigrationConfigRepositoryImpl, mergeUtil, gitOpsConfigRepositoryImpl, gitFactory, applicationServiceClientImpl, argoClientWrapperServiceImpl, pipelineConfigListenerServiceImpl, customTagServiceImpl, acdConfig, deployedAppMetricsServiceImpl) deploymentGroupAppRepositoryImpl := repository.NewDeploymentGroupAppRepositoryImpl(sugaredLogger, db) deploymentGroupServiceImpl := deploymentGroup.NewDeploymentGroupServiceImpl(appRepositoryImpl, sugaredLogger, pipelineRepositoryImpl, ciPipelineRepositoryImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, deploymentGroupAppRepositoryImpl, ciArtifactRepositoryImpl, appWorkflowRepositoryImpl, workflowDagExecutorImpl) - deploymentConfigServiceImpl := pipeline.NewDeploymentConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, pipelineRepositoryImpl, envLevelAppMetricsRepositoryImpl, appLevelMetricsRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, configMapHistoryServiceImpl, chartRefRepositoryImpl, scopedVariableCMCSManagerImpl) + deploymentConfigServiceImpl := pipeline.NewDeploymentConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, pipelineRepositoryImpl, envLevelAppMetricsRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, configMapHistoryServiceImpl, chartRefRepositoryImpl, scopedVariableCMCSManagerImpl, deployedAppMetricsServiceImpl) pipelineTriggerRestHandlerImpl := restHandler.NewPipelineRestHandler(appServiceImpl, userServiceImpl, validate, enforcerImpl, teamServiceImpl, sugaredLogger, enforcerUtilImpl, workflowDagExecutorImpl, deploymentGroupServiceImpl, argoUserServiceImpl, deploymentConfigServiceImpl) sseSSE := sse.NewSSE() pipelineTriggerRouterImpl := router.NewPipelineTriggerRouter(pipelineTriggerRestHandlerImpl, sseSSE) @@ -505,7 +507,7 @@ func InitializeApp() (*App, error) { return nil, err } devtronAppCMCSServiceImpl := pipeline.NewDevtronAppCMCSServiceImpl(sugaredLogger, appServiceImpl, attributesRepositoryImpl) - cdPipelineConfigServiceImpl := pipeline.NewCdPipelineConfigServiceImpl(sugaredLogger, pipelineRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, appRepositoryImpl, appServiceImpl, deploymentGroupRepositoryImpl, ciCdPipelineOrchestratorImpl, appStatusRepositoryImpl, ciPipelineRepositoryImpl, prePostCdScriptHistoryServiceImpl, clusterRepositoryImpl, helmAppServiceImpl, enforcerUtilImpl, gitOpsConfigRepositoryImpl, pipelineStrategyHistoryServiceImpl, chartRepositoryImpl, resourceGroupServiceImpl, chartTemplateServiceImpl, propertiesConfigServiceImpl, appLevelMetricsRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, pipelineDeploymentServiceTypeConfig, applicationServiceClientImpl, customTagServiceImpl, pipelineConfigListenerServiceImpl, devtronAppCMCSServiceImpl, ciPipelineConfigServiceImpl, buildPipelineSwitchServiceImpl, argoClientWrapperServiceImpl) + cdPipelineConfigServiceImpl := pipeline.NewCdPipelineConfigServiceImpl(sugaredLogger, pipelineRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, appRepositoryImpl, appServiceImpl, deploymentGroupRepositoryImpl, ciCdPipelineOrchestratorImpl, appStatusRepositoryImpl, ciPipelineRepositoryImpl, prePostCdScriptHistoryServiceImpl, clusterRepositoryImpl, helmAppServiceImpl, enforcerUtilImpl, gitOpsConfigRepositoryImpl, pipelineStrategyHistoryServiceImpl, chartRepositoryImpl, resourceGroupServiceImpl, chartTemplateServiceImpl, propertiesConfigServiceImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, pipelineDeploymentServiceTypeConfig, applicationServiceClientImpl, customTagServiceImpl, pipelineConfigListenerServiceImpl, devtronAppCMCSServiceImpl, ciPipelineConfigServiceImpl, buildPipelineSwitchServiceImpl, argoClientWrapperServiceImpl, deployedAppMetricsServiceImpl) appArtifactManagerImpl := pipeline.NewAppArtifactManagerImpl(sugaredLogger, cdWorkflowRepositoryImpl, userServiceImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, ciWorkflowRepositoryImpl, pipelineStageServiceImpl, cdPipelineConfigServiceImpl, dockerArtifactStoreRepositoryImpl, ciPipelineRepositoryImpl, ciTemplateServiceImpl) globalStrategyMetadataChartRefMappingRepositoryImpl := chartRepoRepository.NewGlobalStrategyMetadataChartRefMappingRepositoryImpl(db, sugaredLogger) devtronAppStrategyServiceImpl := pipeline.NewDevtronAppStrategyServiceImpl(sugaredLogger, chartRepositoryImpl, globalStrategyMetadataChartRefMappingRepositoryImpl, ciCdPipelineOrchestratorImpl, cdPipelineConfigServiceImpl) From e42180bfd646a52b569ff86a7a45a41806c61e6f Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Thu, 11 Jan 2024 20:25:07 +0530 Subject: [PATCH 03/64] migrated envLevel app metrics code to new service --- api/restHandler/CoreAppRestHandler.go | 17 +- .../app/DeploymentPipelineRestHandler.go | 62 +---- .../app/PipelineConfigRestHandler.go | 6 +- api/router/PipelineConfigRouter.go | 2 - .../EnvLevelAppMetricsRepository.go | 3 +- pkg/app/AppListingService.go | 50 +--- pkg/bulkAction/BulkUpdateService.go | 41 +--- pkg/chart/ChartService.go | 31 +-- .../deployedAppMetrics/DeployedAppMetrics.go | 122 +++++++++- pkg/pipeline/DeploymentConfigService.go | 61 ++--- .../DeploymentPipelineConfigService.go | 22 +- pkg/pipeline/PropertiesConfig.go | 224 ++++-------------- pkg/pipeline/WorkflowDagExecutor.go | 20 +- .../DeploymentTemplateHistoryService.go | 29 +-- wire_gen.go | 18 +- 15 files changed, 251 insertions(+), 457 deletions(-) diff --git a/api/restHandler/CoreAppRestHandler.go b/api/restHandler/CoreAppRestHandler.go index 6d7f20171f..5a0ccf43f4 100644 --- a/api/restHandler/CoreAppRestHandler.go +++ b/api/restHandler/CoreAppRestHandler.go @@ -1857,12 +1857,13 @@ func (handler CoreAppRestHandlerImpl) createEnvDeploymentTemplate(appId int, use appMetrics = *envConfigProperties.AppMetrics } chartEntry.GlobalOverride = string(envConfigProperties.EnvOverrideValues) - _, err = handler.propertiesConfigService.CreateIfRequired(chartEntry, envId, userId, envConfigProperties.ManualReviewed, models.CHARTSTATUS_SUCCESS, + _, updatedAppMetrics, err := handler.propertiesConfigService.CreateIfRequired(chartEntry, envId, userId, envConfigProperties.ManualReviewed, models.CHARTSTATUS_SUCCESS, true, appMetrics, envConfigProperties.Namespace, envConfigProperties.IsBasicViewLocked, envConfigProperties.CurrentViewEditor, nil) if err != nil { handler.logger.Errorw("service err, CreateIfRequired", "err", err, "appId", appId, "envId", envId, "chartRefId", chartRefId) return err } + envConfigProperties.AppMetrics = &updatedAppMetrics //getting environment properties for db table id(this properties get created when cd pipeline is created) env, err := handler.propertiesConfigService.GetEnvironmentProperties(appId, envId, deploymentTemplateOverride.ChartRefId) @@ -1879,20 +1880,6 @@ func (handler CoreAppRestHandlerImpl) createEnvDeploymentTemplate(appId int, use handler.logger.Errorw("service err, EnvConfigOverrideUpdate", "err", err, "appId", appId, "envId", envId) return err } - - //updating app metrics - appMetricsRequest := &chart.AppMetricEnableDisableRequest{ - AppId: appId, - UserId: userId, - EnvironmentId: envId, - IsAppMetricsEnabled: deploymentTemplateOverride.ShowAppMetrics, - } - _, err = handler.propertiesConfigService.EnvMetricsEnableDisable(appMetricsRequest) - if err != nil { - handler.logger.Errorw("service err, EnvMetricsEnableDisable", "err", err, "appId", appId, "envId", envId) - return err - } - return nil } diff --git a/api/restHandler/app/DeploymentPipelineRestHandler.go b/api/restHandler/app/DeploymentPipelineRestHandler.go index c7dc4f7da6..6516880e78 100644 --- a/api/restHandler/app/DeploymentPipelineRestHandler.go +++ b/api/restHandler/app/DeploymentPipelineRestHandler.go @@ -80,8 +80,6 @@ type DevtronAppDeploymentConfigRestHandler interface { GetDeploymentPipelineStrategy(w http.ResponseWriter, r *http.Request) GetDefaultDeploymentPipelineStrategy(w http.ResponseWriter, r *http.Request) - EnvMetricsEnableDisable(w http.ResponseWriter, r *http.Request) - EnvConfigOverrideCreateNamespace(w http.ResponseWriter, r *http.Request) } @@ -564,7 +562,7 @@ func (handler PipelineConfigRestHandlerImpl) ChangeChartRef(w http.ResponseWrite } } - envMetrics, err := handler.propertiesConfigService.FindEnvLevelAppMetricsByAppIdAndEnvId(request.AppId, request.EnvId) + envMetrics, err := handler.deployedAppMetricsService.GetMetricsFlagByAppIdAndEnvId(request.AppId, request.EnvId) if err != nil { handler.Logger.Errorw("could not find envMetrics for, ChangeChartRef", "err", err, "payload", request) common.WriteJsonResp(w, err, "env metric could not be fetched", http.StatusBadRequest) @@ -573,7 +571,7 @@ func (handler PipelineConfigRestHandlerImpl) ChangeChartRef(w http.ResponseWrite envConfigProperties.ChartRefId = request.TargetChartRefId envConfigProperties.UserId = userId envConfigProperties.EnvironmentId = request.EnvId - envConfigProperties.AppMetrics = envMetrics.AppMetrics + envConfigProperties.AppMetrics = &envMetrics token := r.Header.Get("token") handler.Logger.Infow("request payload, EnvConfigOverrideCreate", "payload", request) @@ -635,7 +633,7 @@ func (handler PipelineConfigRestHandlerImpl) ChangeChartRef(w http.ResponseWrite ctx = context.WithValue(r.Context(), "token", acdToken) appMetrics := false if envConfigProperties.AppMetrics != nil { - appMetrics = *envMetrics.AppMetrics + appMetrics = envMetrics } templateRequest := chart.TemplateRequest{ AppId: request.AppId, @@ -1699,60 +1697,6 @@ func (handler PipelineConfigRestHandlerImpl) EnvConfigOverrideReset(w http.Respo common.WriteJsonResp(w, err, isSuccess, http.StatusOK) } -func (handler PipelineConfigRestHandlerImpl) EnvMetricsEnableDisable(w http.ResponseWriter, r *http.Request) { - userId, err := handler.userAuthService.GetLoggedInUser(r) - if userId == 0 || err != nil { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return - } - token := r.Header.Get("token") - vars := mux.Vars(r) - appId, err := strconv.Atoi(vars["appId"]) - if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - environmentId, err := strconv.Atoi(vars["environmentId"]) - if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - decoder := json.NewDecoder(r.Body) - var appMetricEnableDisableRequest chart.AppMetricEnableDisableRequest - err = decoder.Decode(&appMetricEnableDisableRequest) - appMetricEnableDisableRequest.UserId = userId - appMetricEnableDisableRequest.AppId = appId - appMetricEnableDisableRequest.EnvironmentId = environmentId - if err != nil { - handler.Logger.Errorw("request err, EnvMetricsEnableDisable", "err", err, "appId", appId, "environmentId", environmentId, "payload", appMetricEnableDisableRequest) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - handler.Logger.Infow("request payload, EnvMetricsEnableDisable", "err", err, "appId", appId, "environmentId", environmentId, "payload", appMetricEnableDisableRequest) - app, err := handler.pipelineBuilder.GetApp(appId) - if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - resourceName := handler.enforcerUtil.GetAppRBACNameByAppId(appId) - if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, resourceName); !ok { - common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden) - return - } - object := handler.enforcerUtil.GetAppRBACByAppNameAndEnvId(app.AppName, appMetricEnableDisableRequest.EnvironmentId) - if ok := handler.enforcer.Enforce(token, casbin.ResourceEnvironment, casbin.ActionCreate, object); !ok { - common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden) - return - } - createResp, err := handler.propertiesConfigService.EnvMetricsEnableDisable(&appMetricEnableDisableRequest) - if err != nil { - handler.Logger.Errorw("service err, EnvMetricsEnableDisable", "err", err, "appId", appId, "environmentId", environmentId, "payload", appMetricEnableDisableRequest) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - common.WriteJsonResp(w, err, createResp, http.StatusOK) -} - func (handler *PipelineConfigRestHandlerImpl) ListDeploymentHistory(w http.ResponseWriter, r *http.Request) { userId, err := handler.userAuthService.GetLoggedInUser(r) if userId == 0 || err != nil { diff --git a/api/restHandler/app/PipelineConfigRestHandler.go b/api/restHandler/app/PipelineConfigRestHandler.go index cf88641e43..13ff4f3e37 100644 --- a/api/restHandler/app/PipelineConfigRestHandler.go +++ b/api/restHandler/app/PipelineConfigRestHandler.go @@ -22,6 +22,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" "io" "net/http" "strconv" @@ -132,6 +133,7 @@ type PipelineConfigRestHandlerImpl struct { deploymentTemplateService generateManifest.DeploymentTemplateService pipelineRestHandlerEnvConfig *PipelineRestHandlerEnvConfig ciArtifactRepository repository.CiArtifactRepository + deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService } func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger *zap.SugaredLogger, @@ -156,7 +158,8 @@ func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger scanResultRepository security.ImageScanResultRepository, gitProviderRepo repository.GitProviderRepository, argoUserService argo.ArgoUserService, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, imageTaggingService pipeline.ImageTaggingService, - ciArtifactRepository repository.CiArtifactRepository) *PipelineConfigRestHandlerImpl { + ciArtifactRepository repository.CiArtifactRepository, + deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService) *PipelineConfigRestHandlerImpl { envConfig := &PipelineRestHandlerEnvConfig{} err := env.Parse(envConfig) if err != nil { @@ -194,6 +197,7 @@ func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger deploymentTemplateService: deploymentTemplateService, pipelineRestHandlerEnvConfig: envConfig, ciArtifactRepository: ciArtifactRepository, + deployedAppMetricsService: deployedAppMetricsService, } } diff --git a/api/router/PipelineConfigRouter.go b/api/router/PipelineConfigRouter.go index 2fb48b3d8d..c28c2364ee 100644 --- a/api/router/PipelineConfigRouter.go +++ b/api/router/PipelineConfigRouter.go @@ -134,8 +134,6 @@ func (router PipelineConfigRouterImpl) initPipelineConfigRouter(configRouter *mu configRouter.Path("/env/reset/{appId}/{environmentId}/{id}").HandlerFunc(router.restHandler.EnvConfigOverrideReset).Methods("DELETE") configRouter.Path("/env/namespace/{appId}/{environmentId}").HandlerFunc(router.restHandler.EnvConfigOverrideCreateNamespace).Methods("POST") - configRouter.Path("/env/metrics/{appId}/{environmentId}").HandlerFunc(router.restHandler.EnvMetricsEnableDisable).Methods("POST") - configRouter.Path("/app-wf"). HandlerFunc(router.appWorkflowRestHandler.CreateAppWorkflow).Methods("POST") diff --git a/internal/sql/repository/EnvLevelAppMetricsRepository.go b/internal/sql/repository/EnvLevelAppMetricsRepository.go index c26ce4146e..f849212fab 100644 --- a/internal/sql/repository/EnvLevelAppMetricsRepository.go +++ b/internal/sql/repository/EnvLevelAppMetricsRepository.go @@ -57,7 +57,8 @@ func (impl *EnvLevelAppMetricsRepositoryImpl) Save(metrics *EnvLevelAppMetrics) func (impl *EnvLevelAppMetricsRepositoryImpl) FindByAppIdAndEnvId(appId int, envId int) (*EnvLevelAppMetrics, error) { envAppLevelMetrics := &EnvLevelAppMetrics{} - err := impl.dbConnection.Model(envAppLevelMetrics).Where("env_level_app_metrics.app_id = ? ", appId).Where("env_level_app_metrics.env_id = ? ", envId).Select() + err := impl.dbConnection.Model(envAppLevelMetrics).Where("env_level_app_metrics.app_id = ? ", appId). + Where("env_level_app_metrics.env_id = ? ", envId).Select() return envAppLevelMetrics, err } func (impl *EnvLevelAppMetricsRepositoryImpl) FindByAppId(appId int) ([]*EnvLevelAppMetrics, error) { diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index f93bd4408a..207bec1fd3 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -160,7 +160,6 @@ type AppListingServiceImpl struct { cdWorkflowRepository pipelineConfig.CdWorkflowRepository linkoutsRepository repository.LinkoutsRepository appLevelMetricsRepository repository.AppLevelMetricsRepository - envLevelMetricsRepository repository.EnvLevelAppMetricsRepository pipelineOverrideRepository chartConfig.PipelineOverrideRepository environmentRepository repository2.EnvironmentRepository argoUserService argo.ArgoUserService @@ -175,7 +174,7 @@ func NewAppListingServiceImpl(Logger *zap.SugaredLogger, appListingRepository re application application2.ServiceClient, appRepository app.AppRepository, appListingViewBuilder AppListingViewBuilder, pipelineRepository pipelineConfig.PipelineRepository, linkoutsRepository repository.LinkoutsRepository, appLevelMetricsRepository repository.AppLevelMetricsRepository, - envLevelMetricsRepository repository.EnvLevelAppMetricsRepository, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, + cdWorkflowRepository pipelineConfig.CdWorkflowRepository, pipelineOverrideRepository chartConfig.PipelineOverrideRepository, environmentRepository repository2.EnvironmentRepository, argoUserService argo.ArgoUserService, envOverrideRepository chartConfig.EnvConfigOverrideRepository, chartRepository chartRepoRepository.ChartRepository, ciPipelineRepository pipelineConfig.CiPipelineRepository, @@ -189,7 +188,6 @@ func NewAppListingServiceImpl(Logger *zap.SugaredLogger, appListingRepository re pipelineRepository: pipelineRepository, linkoutsRepository: linkoutsRepository, appLevelMetricsRepository: appLevelMetricsRepository, - envLevelMetricsRepository: envLevelMetricsRepository, cdWorkflowRepository: cdWorkflowRepository, pipelineOverrideRepository: pipelineOverrideRepository, environmentRepository: environmentRepository, @@ -885,52 +883,6 @@ func (impl AppListingServiceImpl) FetchAppDetails(ctx context.Context, appId int return appDetailContainer, nil } -func (impl AppListingServiceImpl) fetchAppAndEnvLevelMatrics(ctx context.Context, appId int, appDetailContainer bean.AppDetailContainer) (bean.AppDetailContainer, error) { - var appMetrics bool - var infraMetrics bool - _, span := otel.Tracer("orchestrator").Start(ctx, "appLevelMetricsRepository.FindByAppId") - appLevelMetrics, err := impl.appLevelMetricsRepository.FindByAppId(appId) - span.End() - if err != nil && err != pg.ErrNoRows { - impl.Logger.Errorw("error in app metrics app level flag", "error", err) - return bean.AppDetailContainer{}, err - } else if appLevelMetrics != nil { - appMetrics = appLevelMetrics.AppMetrics - infraMetrics = appLevelMetrics.InfraMetrics - } - i := 0 - var envIds []int - for _, env := range appDetailContainer.Environments { - envIds = append(envIds, env.EnvironmentId) - } - envLevelAppMetricsMap := make(map[int]*repository.EnvLevelAppMetrics) - _, span = otel.Tracer("orchestrator").Start(ctx, "appLevelMetricsRepository.FindByAppIdAndEnvIds") - envLevelAppMetrics, err := impl.envLevelMetricsRepository.FindByAppIdAndEnvIds(appId, envIds) - span.End() - for _, envLevelAppMetric := range envLevelAppMetrics { - envLevelAppMetricsMap[envLevelAppMetric.EnvId] = envLevelAppMetric - } - for _, env := range appDetailContainer.Environments { - var envLevelMetrics *bool - var envLevelInfraMetrics *bool - envLevelAppMetrics := envLevelAppMetricsMap[env.EnvironmentId] - if envLevelAppMetrics != nil && envLevelAppMetrics.Id != 0 && envLevelAppMetrics.AppMetrics != nil { - envLevelMetrics = envLevelAppMetrics.AppMetrics - } else { - envLevelMetrics = &appMetrics - } - if envLevelAppMetrics != nil && envLevelAppMetrics.Id != 0 && envLevelAppMetrics.InfraMetrics != nil { - envLevelInfraMetrics = envLevelAppMetrics.InfraMetrics - } else { - envLevelInfraMetrics = &infraMetrics - } - appDetailContainer.Environments[i].AppMetrics = envLevelMetrics - appDetailContainer.Environments[i].InfraMetrics = envLevelInfraMetrics - i++ - } - return appDetailContainer, nil -} - func (impl AppListingServiceImpl) setIpAccessProvidedData(ctx context.Context, appDetailContainer bean.AppDetailContainer, clusterId int, isVirtualEnv bool) (bean.AppDetailContainer, error) { ciPipelineId := appDetailContainer.CiPipelineId if ciPipelineId > 0 { diff --git a/pkg/bulkAction/BulkUpdateService.go b/pkg/bulkAction/BulkUpdateService.go index c60c415a0b..9991ca32ee 100644 --- a/pkg/bulkAction/BulkUpdateService.go +++ b/pkg/bulkAction/BulkUpdateService.go @@ -14,7 +14,6 @@ import ( "github.com/devtron-labs/devtron/client/argocdServer/application" "github.com/devtron-labs/devtron/client/argocdServer/repository" "github.com/devtron-labs/devtron/internal/sql/models" - repository3 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/appWorkflow" "github.com/devtron-labs/devtron/internal/sql/repository/bulkUpdate" @@ -26,6 +25,7 @@ import ( "github.com/devtron-labs/devtron/pkg/chart" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" "github.com/devtron-labs/devtron/pkg/pipeline" pipeline1 "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/history" @@ -77,8 +77,6 @@ type BulkUpdateServiceImpl struct { configMapRepository chartConfig.ConfigMapRepository environmentRepository repository2.EnvironmentRepository pipelineRepository pipelineConfig.PipelineRepository - appLevelMetricsRepository repository3.AppLevelMetricsRepository - envLevelAppMetricsRepository repository3.EnvLevelAppMetricsRepository client *http.Client appRepository app.AppRepository deploymentTemplateHistoryService history.DeploymentTemplateHistoryService @@ -96,6 +94,7 @@ type BulkUpdateServiceImpl struct { pubsubClient *pubsub.PubSubClientServiceImpl argoUserService argo.ArgoUserService scopedVariableManager variables.ScopedVariableManager + deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService } func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateRepository, @@ -112,8 +111,6 @@ func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateReposito configMapRepository chartConfig.ConfigMapRepository, environmentRepository repository2.EnvironmentRepository, pipelineRepository pipelineConfig.PipelineRepository, - appLevelMetricsRepository repository3.AppLevelMetricsRepository, - envLevelAppMetricsRepository repository3.EnvLevelAppMetricsRepository, client *http.Client, appRepository app.AppRepository, deploymentTemplateHistoryService history.DeploymentTemplateHistoryService, @@ -127,7 +124,7 @@ func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateReposito pubsubClient *pubsub.PubSubClientServiceImpl, argoUserService argo.ArgoUserService, scopedVariableManager variables.ScopedVariableManager, -) (*BulkUpdateServiceImpl, error) { + deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService) (*BulkUpdateServiceImpl, error) { impl := &BulkUpdateServiceImpl{ bulkUpdateRepository: bulkUpdateRepository, chartRepository: chartRepository, @@ -143,8 +140,6 @@ func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateReposito configMapRepository: configMapRepository, environmentRepository: environmentRepository, pipelineRepository: pipelineRepository, - appLevelMetricsRepository: appLevelMetricsRepository, - envLevelAppMetricsRepository: envLevelAppMetricsRepository, client: client, appRepository: appRepository, deploymentTemplateHistoryService: deploymentTemplateHistoryService, @@ -162,6 +157,7 @@ func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateReposito pubsubClient: pubsubClient, argoUserService: argoUserService, scopedVariableManager: scopedVariableManager, + deployedAppMetricsService: deployedAppMetricsService, } err := impl.SubscribeToCdBulkTriggerTopic() @@ -457,12 +453,10 @@ func (impl BulkUpdateServiceImpl) BulkUpdateDeploymentTemplate(bulkUpdatePayload deploymentTemplateBulkUpdateResponse.Successful = append(deploymentTemplateBulkUpdateResponse.Successful, bulkUpdateSuccessResponse) //creating history entry for deployment template - appLevelAppMetricsEnabled := false - appLevelMetrics, err := impl.appLevelMetricsRepository.FindByAppId(chart.AppId) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in getting app level metrics app level", "error", err) - } else if err == nil { - appLevelAppMetricsEnabled = appLevelMetrics.AppMetrics + appLevelAppMetricsEnabled, err := impl.deployedAppMetricsService.GetMetricsFlagByAppIdEvenIfNotInDb(chart.AppId) + if err != nil { + impl.logger.Errorw("error in getting app level metrics app level", "error", err, "appId", chart.AppId) + return nil } chart.GlobalOverride = modified chart.Values = modified @@ -525,22 +519,13 @@ func (impl BulkUpdateServiceImpl) BulkUpdateDeploymentTemplate(bulkUpdatePayload deploymentTemplateBulkUpdateResponse.Successful = append(deploymentTemplateBulkUpdateResponse.Successful, bulkUpdateSuccessResponse) //creating history entry for deployment template - envLevelAppMetricsEnabled := false - envLevelAppMetrics, err := impl.envLevelAppMetricsRepository.FindByAppIdAndEnvId(chartEnv.Chart.AppId, chartEnv.TargetEnvironment) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in getting env level app metrics", "err", err, "appId", chartEnv.Chart.AppId, "envId", chartEnv.TargetEnvironment) - } else if err == pg.ErrNoRows { - appLevelAppMetrics, err := impl.appLevelMetricsRepository.FindByAppId(chartEnv.Chart.AppId) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in getting app level app metrics", "err", err, "appId", chartEnv.Chart.AppId) - } else if err == nil { - envLevelAppMetricsEnabled = appLevelAppMetrics.AppMetrics - } - } else { - envLevelAppMetricsEnabled = *envLevelAppMetrics.AppMetrics + isAppMetricsEnabled, err := impl.deployedAppMetricsService.GetMetricsFlagForAPipelineByAppIdAndEnvId(chartEnv.Chart.AppId, chartEnv.TargetEnvironment) + if err != nil { + impl.logger.Errorw("error, GetMetricsFlagForAPipelineByAppIdAndEnvId", "err", err, "appId", chartEnv.Chart.AppId, "envId", chartEnv.TargetEnvironment) + return nil } chartEnv.EnvOverrideValues = modified - err = impl.deploymentTemplateHistoryService.CreateDeploymentTemplateHistoryFromEnvOverrideTemplate(chartEnv, nil, envLevelAppMetricsEnabled, 0) + err = impl.deploymentTemplateHistoryService.CreateDeploymentTemplateHistoryFromEnvOverrideTemplate(chartEnv, nil, isAppMetricsEnabled, 0) if err != nil { impl.logger.Errorw("error in creating entry for env deployment template history", "err", err, "envOverride", chartEnv) } diff --git a/pkg/chart/ChartService.go b/pkg/chart/ChartService.go index ca6397f113..ceb5956051 100644 --- a/pkg/chart/ChartService.go +++ b/pkg/chart/ChartService.go @@ -54,7 +54,6 @@ import ( "github.com/devtron-labs/devtron/client/argocdServer/repository" "github.com/devtron-labs/devtron/internal/sql/models" - repository3 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" @@ -116,7 +115,6 @@ type ChartServiceImpl struct { configMapRepository chartConfig.ConfigMapRepository environmentRepository repository4.EnvironmentRepository pipelineRepository pipelineConfig.PipelineRepository - envLevelAppMetricsRepository repository3.EnvLevelAppMetricsRepository client *http.Client deploymentTemplateHistoryService history.DeploymentTemplateHistoryService scopedVariableManager variables.ScopedVariableManager @@ -138,7 +136,6 @@ func NewChartServiceImpl(chartRepository chartRepoRepository.ChartRepository, configMapRepository chartConfig.ConfigMapRepository, environmentRepository repository4.EnvironmentRepository, pipelineRepository pipelineConfig.PipelineRepository, - envLevelAppMetricsRepository repository3.EnvLevelAppMetricsRepository, client *http.Client, deploymentTemplateHistoryService history.DeploymentTemplateHistoryService, scopedVariableManager variables.ScopedVariableManager, @@ -147,7 +144,6 @@ func NewChartServiceImpl(chartRepository chartRepoRepository.ChartRepository, // cache devtron reference charts list devtronChartList, _ := chartRefRepository.FetchAllChartInfoByUploadFlag(false) SetReservedChartList(devtronChartList) - return &ChartServiceImpl{ chartRepository: chartRepository, logger: logger, @@ -164,7 +160,6 @@ func NewChartServiceImpl(chartRepository chartRepoRepository.ChartRepository, configMapRepository: configMapRepository, environmentRepository: environmentRepository, pipelineRepository: pipelineRepository, - envLevelAppMetricsRepository: envLevelAppMetricsRepository, client: client, deploymentTemplateHistoryService: deploymentTemplateHistoryService, scopedVariableManager: scopedVariableManager, @@ -449,9 +444,9 @@ func (impl ChartServiceImpl) Create(templateRequest TemplateRequest, ctx context ChartRefId: templateRequest.ChartRefId, UserId: templateRequest.UserId, } - err = impl.deployedAppMetricsService.CheckAndUpdateAppLevelMetrics(ctx, appLevelMetricsUpdateReq) + err = impl.deployedAppMetricsService.CheckAndUpdateAppOrEnvLevelMetrics(ctx, appLevelMetricsUpdateReq) if err != nil { - impl.logger.Errorw("error, CheckAndUpdateAppLevelMetrics", "err", err, "req", appLevelMetricsUpdateReq) + impl.logger.Errorw("error, CheckAndUpdateAppOrEnvLevelMetrics", "err", err, "req", appLevelMetricsUpdateReq) return nil, err } @@ -846,14 +841,14 @@ func (impl ChartServiceImpl) UpdateAppOverride(ctx context.Context, templateRequ ChartRefId: templateRequest.ChartRefId, UserId: templateRequest.UserId, } - err = impl.deployedAppMetricsService.CheckAndUpdateAppLevelMetrics(ctx, appLevelMetricsUpdateReq) + err = impl.deployedAppMetricsService.CheckAndUpdateAppOrEnvLevelMetrics(ctx, appLevelMetricsUpdateReq) if err != nil { - impl.logger.Errorw("error, CheckAndUpdateAppLevelMetrics", "err", err, "req", appLevelMetricsUpdateReq) + impl.logger.Errorw("error, CheckAndUpdateAppOrEnvLevelMetrics", "err", err, "req", appLevelMetricsUpdateReq) return nil, err } _, span = otel.Tracer("orchestrator").Start(ctx, "CreateDeploymentTemplateHistoryFromGlobalTemplate") //creating history entry for deployment template - err = impl.deploymentTemplateHistoryService.CreateDeploymentTemplateHistoryFromGlobalTemplate(template, nil, templateRequest.IsAppMetricsEnabled) + err = impl.deploymentTemplateHistoryService.CreateDeploymentTemplateHistoryFromGlobalTemplate(template, nil, appLevelMetricsUpdateReq.EnableMetrics) span.End() if err != nil { impl.logger.Errorw("error in creating entry for deployment template history", "err", err, "chart", template) @@ -1138,20 +1133,10 @@ func (impl ChartServiceImpl) UpgradeForApp(appId int, chartRefId int, newAppOver return false, err } //creating history entry for deployment template - isAppMetricsEnabled := false - envLevelAppMetrics, err := impl.envLevelAppMetricsRepository.FindByAppIdAndEnvId(appId, envOverrideNew.TargetEnvironment) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in getting env level app metrics", "err", err, "appId", appId, "envId", envOverrideNew.TargetEnvironment) + isAppMetricsEnabled, err := impl.deployedAppMetricsService.GetMetricsFlagForAPipelineByAppIdAndEnvId(appId, envOverrideNew.TargetEnvironment) + if err != nil { + impl.logger.Errorw("error, GetMetricsFlagForAPipelineByAppIdAndEnvId", "err", err, "appId", appId, "envId", envOverrideNew.TargetEnvironment) return false, err - } else if err == pg.ErrNoRows { - isAppLevelMetricsEnabled, err := impl.deployedAppMetricsService.GetMetricsFlagByAppIdEvenIfNotInDb(appId) - if err != nil { - impl.logger.Errorw("error, GetMetricsFlagByAppIdEvenIfNotInDb", "err", err, "appId", appId) - return false, err - } - isAppMetricsEnabled = isAppLevelMetricsEnabled - } else { - isAppMetricsEnabled = *envLevelAppMetrics.AppMetrics } err = impl.deploymentTemplateHistoryService.CreateDeploymentTemplateHistoryFromEnvOverrideTemplate(envOverrideNew, nil, isAppMetricsEnabled, 0) if err != nil { diff --git a/pkg/deployment/manifest/deployedAppMetrics/DeployedAppMetrics.go b/pkg/deployment/manifest/deployedAppMetrics/DeployedAppMetrics.go index 2dcf0096d3..b04de22cff 100644 --- a/pkg/deployment/manifest/deployedAppMetrics/DeployedAppMetrics.go +++ b/pkg/deployment/manifest/deployedAppMetrics/DeployedAppMetrics.go @@ -15,7 +15,10 @@ import ( type DeployedAppMetricsService interface { GetMetricsFlagByAppIdEvenIfNotInDb(appId int) (bool, error) - CheckAndUpdateAppLevelMetrics(ctx context.Context, req *bean.DeployedAppMetricsRequest) error + GetMetricsFlagByAppIdAndEnvId(appId, envId int) (bool, error) + GetMetricsFlagForAPipelineByAppIdAndEnvId(appId, envId int) (bool, error) + CheckAndUpdateAppOrEnvLevelMetrics(ctx context.Context, req *bean.DeployedAppMetricsRequest) error + DeleteEnvLevelMetricsIfPresent(appId, envId int) error } type DeployedAppMetricsServiceImpl struct { @@ -40,7 +43,7 @@ func NewDeployedAppMetricsServiceImpl(logger *zap.SugaredLogger, func (impl *DeployedAppMetricsServiceImpl) GetMetricsFlagByAppIdEvenIfNotInDb(appId int) (bool, error) { isAppMetricsEnabled := false appMetrics, err := impl.appLevelMetricsRepository.FindByAppId(appId) - if err != nil && !util.IsErrNoRows(err) { + if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in fetching app level metrics", "appId", appId, "err", err) return isAppMetricsEnabled, err } @@ -50,8 +53,41 @@ func (impl *DeployedAppMetricsServiceImpl) GetMetricsFlagByAppIdEvenIfNotInDb(ap return isAppMetricsEnabled, nil } -// CheckAndUpdateAppLevelMetrics - this method checks whether chart being used supports metrics or not and update accordingly -func (impl *DeployedAppMetricsServiceImpl) CheckAndUpdateAppLevelMetrics(ctx context.Context, req *bean.DeployedAppMetricsRequest) error { +func (impl *DeployedAppMetricsServiceImpl) GetMetricsFlagByAppIdAndEnvId(appId, envId int) (bool, error) { + isAppMetricsEnabled := false + envLevelAppMetrics, err := impl.envLevelMetricsRepository.FindByAppIdAndEnvId(appId, envId) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in getting env level app metrics", "err", err, "appId", appId, "envId", envId) + return isAppMetricsEnabled, err + } + if envLevelAppMetrics != nil { + isAppMetricsEnabled = *envLevelAppMetrics.AppMetrics + } + return isAppMetricsEnabled, nil +} + +// GetMetricsFlagForAPipelineByAppIdAndEnvId - this function returns metrics flag for pipeline after resolving override and app level values +func (impl *DeployedAppMetricsServiceImpl) GetMetricsFlagForAPipelineByAppIdAndEnvId(appId, envId int) (bool, error) { + isAppMetricsEnabled := false + envLevelAppMetrics, err := impl.envLevelMetricsRepository.FindByAppIdAndEnvId(appId, envId) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in getting env level app metrics", "err", err, "appId", appId, "envId", envId) + return isAppMetricsEnabled, err + } else if err == pg.ErrNoRows { + isAppLevelMetricsEnabled, err := impl.GetMetricsFlagByAppIdEvenIfNotInDb(appId) + if err != nil { + impl.logger.Errorw("error, GetMetricsFlagByAppIdEvenIfNotInDb", "err", err, "appId", appId) + return false, err + } + isAppMetricsEnabled = isAppLevelMetricsEnabled + } else if envLevelAppMetrics != nil && envLevelAppMetrics.AppMetrics != nil { + isAppMetricsEnabled = *envLevelAppMetrics.AppMetrics + } + return isAppMetricsEnabled, nil +} + +// CheckAndUpdateAppOrEnvLevelMetrics - this method checks whether chart being used supports metrics or not, is app level or env level and updates accordingly +func (impl *DeployedAppMetricsServiceImpl) CheckAndUpdateAppOrEnvLevelMetrics(ctx context.Context, req *bean.DeployedAppMetricsRequest) error { isAppMetricsSupported, err := impl.checkIsAppMetricsSupported(req.ChartRefId) if err != nil { return err @@ -59,14 +95,41 @@ func (impl *DeployedAppMetricsServiceImpl) CheckAndUpdateAppLevelMetrics(ctx con if !(isAppMetricsSupported) { //chart does not have metrics support, disabling req.EnableMetrics = false + return nil } - _, span := otel.Tracer("orchestrator").Start(ctx, "updateAppLevelMetrics") - _, err = impl.updateAppLevelMetrics(req) - span.End() - if err != nil { - impl.logger.Errorw("error in disable app metric flag", "error", err) + if req.EnvId == 0 { + _, span := otel.Tracer("orchestrator").Start(ctx, "createOrUpdateAppLevelMetrics") + _, err = impl.createOrUpdateAppLevelMetrics(req) + span.End() + if err != nil { + impl.logger.Errorw("error in disable app metric flag", "error", err, "req", req) + return err + } + } else { + _, span := otel.Tracer("orchestrator").Start(ctx, "createOrUpdateEnvLevelMetrics") + _, err = impl.createOrUpdateEnvLevelMetrics(req) + span.End() + if err != nil { + impl.logger.Errorw("error in disable env level app metric flag", "error", err, "req", req) + return err + } + } + return nil +} + +func (impl *DeployedAppMetricsServiceImpl) DeleteEnvLevelMetricsIfPresent(appId, envId int) error { + envLevelAppMetrics, err := impl.envLevelMetricsRepository.FindByAppIdAndEnvId(appId, envId) + if err != nil && !util.IsErrNoRows(err) { + impl.logger.Errorw("error while fetching env level app metric", "err", err, "appId", appId, "envId", envId) return err } + if envLevelAppMetrics != nil && envLevelAppMetrics.Id > 0 { + err = impl.envLevelMetricsRepository.Delete(envLevelAppMetrics) + if err != nil { + impl.logger.Errorw("error while deletion of app metric at env level", "err", err, "model", envLevelAppMetrics) + return err + } + } return nil } @@ -79,7 +142,7 @@ func (impl *DeployedAppMetricsServiceImpl) checkIsAppMetricsSupported(chartRefId return chartRefValue.IsAppMetricsSupported, nil } -func (impl *DeployedAppMetricsServiceImpl) updateAppLevelMetrics(req *bean.DeployedAppMetricsRequest) (*interalRepo.AppLevelMetrics, error) { +func (impl *DeployedAppMetricsServiceImpl) createOrUpdateAppLevelMetrics(req *bean.DeployedAppMetricsRequest) (*interalRepo.AppLevelMetrics, error) { existingAppLevelMetrics, err := impl.appLevelMetricsRepository.FindByAppId(req.AppId) if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in app metrics app level flag", "error", err) @@ -115,3 +178,42 @@ func (impl *DeployedAppMetricsServiceImpl) updateAppLevelMetrics(req *bean.Deplo return appLevelMetricsNew, nil } } + +func (impl *DeployedAppMetricsServiceImpl) createOrUpdateEnvLevelMetrics(req *bean.DeployedAppMetricsRequest) (*interalRepo.EnvLevelAppMetrics, error) { + // update and create env level app metrics + envLevelAppMetrics, err := impl.envLevelMetricsRepository.FindByAppIdAndEnvId(req.AppId, req.EnvId) + if err != nil && err != pg.ErrNoRows { + impl.logger.Error("err", err) + return nil, err + } + if envLevelAppMetrics == nil || envLevelAppMetrics.Id == 0 { + infraMetrics := true + envLevelAppMetrics = &interalRepo.EnvLevelAppMetrics{ + AppId: req.AppId, + EnvId: req.EnvId, + AppMetrics: &req.EnableMetrics, + InfraMetrics: &infraMetrics, + AuditLog: sql.AuditLog{ + CreatedOn: time.Now(), + UpdatedOn: time.Now(), + CreatedBy: req.UserId, + UpdatedBy: req.UserId, + }, + } + err = impl.envLevelMetricsRepository.Save(envLevelAppMetrics) + if err != nil { + impl.logger.Error("err", err) + return nil, err + } + } else { + envLevelAppMetrics.AppMetrics = &req.EnableMetrics + envLevelAppMetrics.UpdatedOn = time.Now() + envLevelAppMetrics.UpdatedBy = req.UserId + err = impl.envLevelMetricsRepository.Update(envLevelAppMetrics) + if err != nil { + impl.logger.Error("err", err) + return nil, err + } + } + return envLevelAppMetrics, err +} diff --git a/pkg/pipeline/DeploymentConfigService.go b/pkg/pipeline/DeploymentConfigService.go index 673bc39365..1e5a57c615 100644 --- a/pkg/pipeline/DeploymentConfigService.go +++ b/pkg/pipeline/DeploymentConfigService.go @@ -3,7 +3,6 @@ package pipeline import ( "context" "encoding/json" - "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/pkg/bean" @@ -25,24 +24,22 @@ type DeploymentConfigService interface { } type DeploymentConfigServiceImpl struct { - logger *zap.SugaredLogger - envConfigOverrideRepository chartConfig.EnvConfigOverrideRepository - chartRepository chartRepoRepository.ChartRepository - pipelineRepository pipelineConfig.PipelineRepository - envLevelAppMetricsRepository repository.EnvLevelAppMetricsRepository - pipelineConfigRepository chartConfig.PipelineConfigRepository - configMapRepository chartConfig.ConfigMapRepository - configMapHistoryService history.ConfigMapHistoryService - chartRefRepository chartRepoRepository.ChartRefRepository - scopedVariableManager variables.ScopedVariableCMCSManager - deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService + logger *zap.SugaredLogger + envConfigOverrideRepository chartConfig.EnvConfigOverrideRepository + chartRepository chartRepoRepository.ChartRepository + pipelineRepository pipelineConfig.PipelineRepository + pipelineConfigRepository chartConfig.PipelineConfigRepository + configMapRepository chartConfig.ConfigMapRepository + configMapHistoryService history.ConfigMapHistoryService + chartRefRepository chartRepoRepository.ChartRefRepository + scopedVariableManager variables.ScopedVariableCMCSManager + deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService } func NewDeploymentConfigServiceImpl(logger *zap.SugaredLogger, envConfigOverrideRepository chartConfig.EnvConfigOverrideRepository, chartRepository chartRepoRepository.ChartRepository, pipelineRepository pipelineConfig.PipelineRepository, - envLevelAppMetricsRepository repository.EnvLevelAppMetricsRepository, pipelineConfigRepository chartConfig.PipelineConfigRepository, configMapRepository chartConfig.ConfigMapRepository, configMapHistoryService history.ConfigMapHistoryService, @@ -50,17 +47,16 @@ func NewDeploymentConfigServiceImpl(logger *zap.SugaredLogger, scopedVariableManager variables.ScopedVariableCMCSManager, deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService) *DeploymentConfigServiceImpl { return &DeploymentConfigServiceImpl{ - logger: logger, - envConfigOverrideRepository: envConfigOverrideRepository, - chartRepository: chartRepository, - pipelineRepository: pipelineRepository, - envLevelAppMetricsRepository: envLevelAppMetricsRepository, - pipelineConfigRepository: pipelineConfigRepository, - configMapRepository: configMapRepository, - configMapHistoryService: configMapHistoryService, - chartRefRepository: chartRefRepository, - scopedVariableManager: scopedVariableManager, - deployedAppMetricsService: deployedAppMetricsService, + logger: logger, + envConfigOverrideRepository: envConfigOverrideRepository, + chartRepository: chartRepository, + pipelineRepository: pipelineRepository, + pipelineConfigRepository: pipelineConfigRepository, + configMapRepository: configMapRepository, + configMapHistoryService: configMapHistoryService, + chartRefRepository: chartRefRepository, + scopedVariableManager: scopedVariableManager, + deployedAppMetricsService: deployedAppMetricsService, } } @@ -97,19 +93,10 @@ func (impl *DeploymentConfigServiceImpl) GetLatestDeploymentConfigurationByPipel } func (impl *DeploymentConfigServiceImpl) GetLatestDeploymentTemplateConfig(ctx context.Context, pipeline *pipelineConfig.Pipeline) (*history.HistoryDetailDto, error) { - isAppMetricsEnabled := false - envLevelAppMetrics, err := impl.envLevelAppMetricsRepository.FindByAppIdAndEnvId(pipeline.AppId, pipeline.EnvironmentId) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in getting env level app metrics", "err", err, "appId", pipeline.AppId, "envId", pipeline.EnvironmentId) - } else if err == pg.ErrNoRows { - isAppLevelMetricsEnabled, err := impl.deployedAppMetricsService.GetMetricsFlagByAppIdEvenIfNotInDb(pipeline.AppId) - if err != nil { - impl.logger.Errorw("error, GetMetricsFlagByAppIdEvenIfNotInDb", "err", err, "appId", pipeline.AppId) - return nil, err - } - isAppMetricsEnabled = isAppLevelMetricsEnabled - } else { - isAppMetricsEnabled = *envLevelAppMetrics.AppMetrics + isAppMetricsEnabled, err := impl.deployedAppMetricsService.GetMetricsFlagForAPipelineByAppIdAndEnvId(pipeline.AppId, pipeline.EnvironmentId) + if err != nil { + impl.logger.Errorw("error, GetMetricsFlagForAPipelineByAppIdAndEnvId", "err", err, "appId", pipeline.AppId, "envId", pipeline.EnvironmentId) + return nil, err } envOverride, err := impl.envConfigOverrideRepository.ActiveEnvConfigOverride(pipeline.AppId, pipeline.EnvironmentId) if err != nil { diff --git a/pkg/pipeline/DeploymentPipelineConfigService.go b/pkg/pipeline/DeploymentPipelineConfigService.go index 8a80c10bd3..d7a3dfd307 100644 --- a/pkg/pipeline/DeploymentPipelineConfigService.go +++ b/pkg/pipeline/DeploymentPipelineConfigService.go @@ -1672,11 +1672,20 @@ func (impl *CdPipelineConfigServiceImpl) createCdPipeline(ctx context.Context, a if err != nil { return 0, err } - envOverride, err := impl.propertiesConfigService.CreateIfRequired(chart, pipeline.EnvironmentId, userId, false, models.CHARTSTATUS_NEW, false, false, pipeline.Namespace, chart.IsBasicViewLocked, chart.CurrentViewEditor, tx) + //getting global app metrics for cd pipeline create because env level metrics is not created yet + appLevelAppMetricsEnabled := false + isAppLevelMetricsEnabled, err := impl.deployedAppMetricsService.GetMetricsFlagByAppIdEvenIfNotInDb(app.Id) if err != nil { + impl.logger.Errorw("error, GetMetricsFlagByAppIdEvenIfNotInDb", "err", err, "appId", app.Id) return 0, err } - + appLevelAppMetricsEnabled = isAppLevelMetricsEnabled + envOverride, updatedAppMetrics, err := impl.propertiesConfigService.CreateIfRequired(chart, pipeline.EnvironmentId, userId, false, + models.CHARTSTATUS_NEW, false, appLevelAppMetricsEnabled, pipeline.Namespace, chart.IsBasicViewLocked, chart.CurrentViewEditor, tx) + if err != nil { + return 0, err + } + appLevelAppMetricsEnabled = updatedAppMetrics // Get pipeline override based on Deployment strategy //TODO: mark as created in our db pipelineId, err = impl.ciCdPipelineOrchestrator.CreateCDPipelines(pipeline, app.Id, userId, tx, app.AppName) @@ -1733,14 +1742,7 @@ func (impl *CdPipelineConfigServiceImpl) createCdPipeline(ctx context.Context, a return 0, err } } - //getting global app metrics for cd pipeline create because env level metrics is not created yet - appLevelAppMetricsEnabled := false - isAppLevelMetricsEnabled, err := impl.deployedAppMetricsService.GetMetricsFlagByAppIdEvenIfNotInDb(app.Id) - if err != nil { - impl.logger.Errorw("error, GetMetricsFlagByAppIdEvenIfNotInDb", "err", err, "appId", app.Id) - return 0, err - } - appLevelAppMetricsEnabled = isAppLevelMetricsEnabled + err = impl.deploymentTemplateHistoryService.CreateDeploymentTemplateHistoryFromEnvOverrideTemplate(envOverride, tx, appLevelAppMetricsEnabled, pipelineId) if err != nil { impl.logger.Errorw("error in creating entry for env deployment template history", "err", err, "envOverride", envOverride) diff --git a/pkg/pipeline/PropertiesConfig.go b/pkg/pipeline/PropertiesConfig.go index e20aaae55d..686042bd65 100644 --- a/pkg/pipeline/PropertiesConfig.go +++ b/pkg/pipeline/PropertiesConfig.go @@ -18,8 +18,11 @@ package pipeline import ( + "context" "encoding/json" "fmt" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" + bean2 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/bean" "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/variables" repository5 "github.com/devtron-labs/devtron/pkg/variables/repository" @@ -33,7 +36,6 @@ import ( "github.com/devtron-labs/devtron/client/argocdServer/application" "github.com/devtron-labs/devtron/internal/sql/models" - "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" "github.com/devtron-labs/devtron/internal/util" "github.com/go-pg/pg" @@ -45,17 +47,15 @@ type PropertiesConfigService interface { CreateEnvironmentProperties(appId int, propertiesRequest *bean.EnvironmentProperties) (*bean.EnvironmentProperties, error) UpdateEnvironmentProperties(appId int, propertiesRequest *bean.EnvironmentProperties, userId int32) (*bean.EnvironmentProperties, error) //create environment entry for each new environment - CreateIfRequired(chart *chartRepoRepository.Chart, environmentId int, userId int32, manualReviewed bool, chartStatus models.ChartStatus, isOverride, isAppMetricsEnabled bool, namespace string, IsBasicViewLocked bool, CurrentViewEditor models.ChartsViewEditorType, tx *pg.Tx) (*chartConfig.EnvConfigOverride, error) + CreateIfRequired(chart *chartRepoRepository.Chart, environmentId int, userId int32, manualReviewed bool, chartStatus models.ChartStatus, isOverride, isAppMetricsEnabled bool, namespace string, IsBasicViewLocked bool, CurrentViewEditor models.ChartsViewEditorType, tx *pg.Tx) (*chartConfig.EnvConfigOverride, bool, error) GetEnvironmentProperties(appId, environmentId int, chartRefId int) (environmentPropertiesResponse *bean.EnvironmentPropertiesResponse, err error) GetEnvironmentPropertiesById(environmentId int) ([]bean.EnvironmentProperties, error) GetAppIdByChartEnvId(chartEnvId int) (*chartConfig.EnvConfigOverride, error) GetLatestEnvironmentProperties(appId, environmentId int) (*bean.EnvironmentProperties, error) - FindEnvLevelAppMetricsByAppIdAndEnvId(appId int, envId int) (*repository.EnvLevelAppMetrics, error) ResetEnvironmentProperties(id int) (bool, error) CreateEnvironmentPropertiesWithNamespace(appId int, propertiesRequest *bean.EnvironmentProperties) (*bean.EnvironmentProperties, error) - EnvMetricsEnableDisable(appMetricRequest *chartService.AppMetricEnableDisableRequest) (*chartService.AppMetricEnableDisableRequest, error) FetchEnvProperties(appId, envId, chartRefId int) (*chartConfig.EnvConfigOverride, error) } type PropertiesConfigServiceImpl struct { @@ -67,10 +67,9 @@ type PropertiesConfigServiceImpl struct { environmentRepository repository2.EnvironmentRepository ciCdPipelineOrchestrator CiCdPipelineOrchestrator application application.ServiceClient - envLevelAppMetricsRepository repository.EnvLevelAppMetricsRepository - appLevelMetricsRepository repository.AppLevelMetricsRepository deploymentTemplateHistoryService history.DeploymentTemplateHistoryService scopedVariableManager variables.ScopedVariableManager + deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService } func NewPropertiesConfigServiceImpl(logger *zap.SugaredLogger, @@ -81,11 +80,9 @@ func NewPropertiesConfigServiceImpl(logger *zap.SugaredLogger, environmentRepository repository2.EnvironmentRepository, ciCdPipelineOrchestrator CiCdPipelineOrchestrator, application application.ServiceClient, - envLevelAppMetricsRepository repository.EnvLevelAppMetricsRepository, - appLevelMetricsRepository repository.AppLevelMetricsRepository, deploymentTemplateHistoryService history.DeploymentTemplateHistoryService, scopedVariableManager variables.ScopedVariableManager, -) *PropertiesConfigServiceImpl { + deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService) *PropertiesConfigServiceImpl { return &PropertiesConfigServiceImpl{ logger: logger, envConfigRepo: envConfigRepo, @@ -95,10 +92,9 @@ func NewPropertiesConfigServiceImpl(logger *zap.SugaredLogger, environmentRepository: environmentRepository, ciCdPipelineOrchestrator: ciCdPipelineOrchestrator, application: application, - envLevelAppMetricsRepository: envLevelAppMetricsRepository, - appLevelMetricsRepository: appLevelMetricsRepository, deploymentTemplateHistoryService: deploymentTemplateHistoryService, scopedVariableManager: scopedVariableManager, + deployedAppMetricsService: deployedAppMetricsService, } } @@ -191,27 +187,12 @@ func (impl PropertiesConfigServiceImpl) GetEnvironmentProperties(appId, environm environmentPropertiesResponse.EnvironmentConfig.CurrentViewEditor = chart.CurrentViewEditor } } - - envLevelMetrics, err := impl.envLevelAppMetricsRepository.FindByAppIdAndEnvId(appId, environmentId) - if err != nil && !util.IsErrNoRows(err) { - impl.logger.Error(err) + isAppMetricsEnabled, err := impl.deployedAppMetricsService.GetMetricsFlagForAPipelineByAppIdAndEnvId(appId, environmentId) + if err != nil { + impl.logger.Errorw("error, GetMetricsFlagForAPipelineByAppIdAndEnvId", "err", err, "appId", appId, "envId", environmentId) return nil, err } - if util.IsErrNoRows(err) { - appLevelMetrics, err := impl.appLevelMetricsRepository.FindByAppId(appId) - if err != nil && !util.IsErrNoRows(err) { - impl.logger.Errorw("error in app metrics app level flag", "error", err) - return nil, err - } - if util.IsErrNoRows(err) { - flag := false - environmentPropertiesResponse.AppMetrics = &flag - } else { - environmentPropertiesResponse.AppMetrics = &appLevelMetrics.AppMetrics - } - } else { - environmentPropertiesResponse.AppMetrics = envLevelMetrics.AppMetrics - } + environmentPropertiesResponse.AppMetrics = &isAppMetricsEnabled return environmentPropertiesResponse, nil } @@ -233,11 +214,11 @@ func (impl PropertiesConfigServiceImpl) CreateEnvironmentProperties(appId int, e if environmentProperties.AppMetrics != nil { appMetrics = *environmentProperties.AppMetrics } - envOverride, err := impl.CreateIfRequired(chart, environmentProperties.EnvironmentId, environmentProperties.UserId, environmentProperties.ManualReviewed, models.CHARTSTATUS_SUCCESS, true, appMetrics, environmentProperties.Namespace, environmentProperties.IsBasicViewLocked, environmentProperties.CurrentViewEditor, nil) + envOverride, appMetrics, err := impl.CreateIfRequired(chart, environmentProperties.EnvironmentId, environmentProperties.UserId, environmentProperties.ManualReviewed, models.CHARTSTATUS_SUCCESS, true, appMetrics, environmentProperties.Namespace, environmentProperties.IsBasicViewLocked, environmentProperties.CurrentViewEditor, nil) if err != nil { return nil, err } - + environmentProperties.AppMetrics = &appMetrics r := json.RawMessage{} err = r.UnmarshalJSON([]byte(envOverride.EnvOverrideValues)) if err != nil { @@ -260,31 +241,10 @@ func (impl PropertiesConfigServiceImpl) CreateEnvironmentProperties(appId int, e ChartRefId: environmentProperties.ChartRefId, IsOverride: envOverride.IsOverride, } - if err != nil { impl.logger.Errorw("chart version parsing", "err", err, "chartVersion", chart.ChartVersion) return nil, err } - chartRefValue, err := impl.chartRefRepository.FindById(environmentProperties.ChartRefId) - if err != nil { - impl.logger.Errorw("error in finding ref chart by id", "err", err) - return nil, err - } - if !(chartRefValue.IsAppMetricsSupported) { - appMetricsRequest := chartService.AppMetricEnableDisableRequest{UserId: environmentProperties.UserId, AppId: appId, EnvironmentId: environmentProperties.EnvironmentId, IsAppMetricsEnabled: false} - _, err = impl.EnvMetricsEnableDisable(&appMetricsRequest) - if err != nil { - impl.logger.Errorw("err while disable app metrics", "err", err, "appId", appId, "chartVersion", chart.ChartVersion) - return nil, err - } - } else { - appMetricsRequest := chartService.AppMetricEnableDisableRequest{UserId: environmentProperties.UserId, AppId: appId, EnvironmentId: environmentProperties.EnvironmentId, IsAppMetricsEnabled: appMetrics} - _, err = impl.EnvMetricsEnableDisable(&appMetricsRequest) - if err != nil { - impl.logger.Errorw("err while updating app metrics", "err", err) - return nil, err - } - } return environmentProperties, nil } @@ -357,26 +317,18 @@ func (impl PropertiesConfigServiceImpl) UpdateEnvironmentProperties(appId int, p if propertiesRequest.AppMetrics != nil { isAppMetricsEnabled = *propertiesRequest.AppMetrics } - chartRefValue, err := impl.chartRefRepository.FindById(oldEnvOverride.Chart.ChartRefId) + envLevelMetricsUpdateReq := &bean2.DeployedAppMetricsRequest{ + EnableMetrics: isAppMetricsEnabled, + AppId: appId, + EnvId: oldEnvOverride.TargetEnvironment, + ChartRefId: oldEnvOverride.Chart.ChartRefId, + UserId: propertiesRequest.UserId, + } + err = impl.deployedAppMetricsService.CheckAndUpdateAppOrEnvLevelMetrics(context.Background(), envLevelMetricsUpdateReq) if err != nil { - impl.logger.Errorw("error in finding ref chart by id", "err", err) + impl.logger.Errorw("error, CheckAndUpdateAppOrEnvLevelMetrics", "err", err, "req", envLevelMetricsUpdateReq) return nil, err } - if !(chartRefValue.IsAppMetricsSupported) { - appMetricsRequest := chartService.AppMetricEnableDisableRequest{UserId: propertiesRequest.UserId, AppId: appId, EnvironmentId: oldEnvOverride.TargetEnvironment, IsAppMetricsEnabled: false} - _, err = impl.EnvMetricsEnableDisable(&appMetricsRequest) - if err != nil { - impl.logger.Errorw("err while disable app metrics for lower versions", err) - return nil, err - } - } else { - appMetricsRequest := chartService.AppMetricEnableDisableRequest{UserId: propertiesRequest.UserId, AppId: appId, EnvironmentId: oldEnvOverride.TargetEnvironment, IsAppMetricsEnabled: isAppMetricsEnabled} - _, err = impl.EnvMetricsEnableDisable(&appMetricsRequest) - if err != nil { - impl.logger.Errorw("err while updating app metrics", "err", err) - return nil, err - } - } //creating history err = impl.deploymentTemplateHistoryService.CreateDeploymentTemplateHistoryFromEnvOverrideTemplate(override, nil, isAppMetricsEnabled, 0) @@ -405,10 +357,10 @@ func (impl PropertiesConfigServiceImpl) buildAppMetricsJson() ([]byte, error) { return appMetricsJson, nil } -func (impl PropertiesConfigServiceImpl) CreateIfRequired(chart *chartRepoRepository.Chart, environmentId int, userId int32, manualReviewed bool, chartStatus models.ChartStatus, isOverride, isAppMetricsEnabled bool, namespace string, IsBasicViewLocked bool, CurrentViewEditor models.ChartsViewEditorType, tx *pg.Tx) (*chartConfig.EnvConfigOverride, error) { +func (impl PropertiesConfigServiceImpl) CreateIfRequired(chart *chartRepoRepository.Chart, environmentId int, userId int32, manualReviewed bool, chartStatus models.ChartStatus, isOverride, isAppMetricsEnabled bool, namespace string, IsBasicViewLocked bool, CurrentViewEditor models.ChartsViewEditorType, tx *pg.Tx) (*chartConfig.EnvConfigOverride, bool, error) { env, err := impl.environmentRepository.FindById(environmentId) if err != nil { - return nil, err + return nil, isAppMetricsEnabled, err } if env != nil && len(env.Namespace) > 0 { @@ -417,14 +369,14 @@ func (impl PropertiesConfigServiceImpl) CreateIfRequired(chart *chartRepoReposit envOverride, err := impl.envConfigRepo.GetByChartAndEnvironment(chart.Id, environmentId) if err != nil && !errors.IsNotFound(err) { - return nil, err + return nil, isAppMetricsEnabled, err } if errors.IsNotFound(err) { if isOverride { // before creating new entry, remove previous one from latest tag envOverrideExisting, err := impl.envConfigRepo.FindLatestChartForAppByAppIdAndEnvId(chart.AppId, environmentId) if err != nil && !errors.IsNotFound(err) { - return nil, err + return nil, isAppMetricsEnabled, err } if envOverrideExisting != nil { envOverrideExisting.Latest = false @@ -440,7 +392,7 @@ func (impl PropertiesConfigServiceImpl) CreateIfRequired(chart *chartRepoReposit envOverrideExisting, err = impl.envConfigRepo.Update(envOverrideExisting) } if err != nil { - return nil, err + return nil, isAppMetricsEnabled, err } } } @@ -473,23 +425,37 @@ func (impl PropertiesConfigServiceImpl) CreateIfRequired(chart *chartRepoReposit } if err != nil { impl.logger.Errorw("error in creating envconfig", "data", envOverride, "error", err) - return nil, err + return nil, isAppMetricsEnabled, err } + envLevelMetricsUpdateReq := &bean2.DeployedAppMetricsRequest{ + EnableMetrics: isAppMetricsEnabled, + AppId: chart.AppId, + EnvId: environmentId, + ChartRefId: chart.ChartRefId, + UserId: userId, + } + err = impl.deployedAppMetricsService.CheckAndUpdateAppOrEnvLevelMetrics(context.Background(), envLevelMetricsUpdateReq) + if err != nil { + impl.logger.Errorw("error, CheckAndUpdateAppOrEnvLevelMetrics", "err", err, "req", envLevelMetricsUpdateReq) + return nil, isAppMetricsEnabled, err + } + //updating metrics flag because it might be possible that the chartRef used was not supported and that could have override the metrics flag got in request + isAppMetricsEnabled = envLevelMetricsUpdateReq.EnableMetrics err = impl.deploymentTemplateHistoryService.CreateDeploymentTemplateHistoryFromEnvOverrideTemplate(envOverride, tx, isAppMetricsEnabled, 0) if err != nil { impl.logger.Errorw("error in creating entry for env deployment template history", "err", err, "envOverride", envOverride) - return nil, err + return nil, isAppMetricsEnabled, err } //VARIABLE_MAPPING_UPDATE if envOverride.EnvOverrideValues != "{}" { err = impl.scopedVariableManager.ExtractAndMapVariables(envOverride.EnvOverrideValues, envOverride.Id, repository5.EntityTypeDeploymentTemplateEnvLevel, envOverride.CreatedBy, tx) if err != nil { - return nil, err + return nil, isAppMetricsEnabled, err } } } - return envOverride, nil + return envOverride, isAppMetricsEnabled, nil } func (impl PropertiesConfigServiceImpl) GetEnvironmentPropertiesById(envId int) ([]bean.EnvironmentProperties, error) { @@ -525,10 +491,6 @@ func (impl PropertiesConfigServiceImpl) GetAppIdByChartEnvId(chartEnvId int) (*c return envOverride, nil } -func (impl PropertiesConfigServiceImpl) FindEnvLevelAppMetricsByAppIdAndEnvId(appId int, envId int) (*repository.EnvLevelAppMetrics, error) { - return impl.envLevelAppMetricsRepository.FindByAppIdAndEnvId(appId, envId) -} - func (impl PropertiesConfigServiceImpl) GetLatestEnvironmentProperties(appId, environmentId int) (environmentProperties *bean.EnvironmentProperties, err error) { env, err := impl.environmentRepository.FindById(environmentId) if err != nil { @@ -583,18 +545,11 @@ func (impl PropertiesConfigServiceImpl) ResetEnvironmentProperties(id int) (bool if err != nil { impl.logger.Warnw("error in update envOverride", "envOverrideId", id) } - envLevelAppMetrics, err := impl.envLevelAppMetricsRepository.FindByAppIdAndEnvId(envOverride.Chart.AppId, envOverride.TargetEnvironment) - if err != nil && !util.IsErrNoRows(err) { - impl.logger.Errorw("error while fetching env level app metric", "err", err) + err = impl.deployedAppMetricsService.DeleteEnvLevelMetricsIfPresent(envOverride.Chart.AppId, envOverride.TargetEnvironment) + if err != nil { + impl.logger.Errorw("error, DeleteEnvLevelMetricsIfPresent", "err", err, "appId", envOverride.Chart.AppId, "envId", envOverride.TargetEnvironment) return false, err } - if envLevelAppMetrics.Id > 0 { - err = impl.envLevelAppMetricsRepository.Delete(envLevelAppMetrics) - if err != nil { - impl.logger.Errorw("error while deletion of app metric at env level", "err", err) - return false, err - } - } //VARIABLES err = impl.scopedVariableManager.RemoveMappedVariables(envOverride.Id, repository5.EntityTypeDeploymentTemplateEnvLevel, envOverride.UpdatedBy, nil) if err != nil { @@ -623,10 +578,11 @@ func (impl PropertiesConfigServiceImpl) CreateEnvironmentPropertiesWithNamespace if environmentProperties.AppMetrics != nil { appMetrics = *environmentProperties.AppMetrics } - envOverride, err = impl.CreateIfRequired(chart, environmentProperties.EnvironmentId, environmentProperties.UserId, environmentProperties.ManualReviewed, models.CHARTSTATUS_SUCCESS, false, appMetrics, environmentProperties.Namespace, environmentProperties.IsBasicViewLocked, environmentProperties.CurrentViewEditor, nil) + envOverride, appMetrics, err = impl.CreateIfRequired(chart, environmentProperties.EnvironmentId, environmentProperties.UserId, environmentProperties.ManualReviewed, models.CHARTSTATUS_SUCCESS, false, appMetrics, environmentProperties.Namespace, environmentProperties.IsBasicViewLocked, environmentProperties.CurrentViewEditor, nil) if err != nil { return nil, err } + environmentProperties.AppMetrics = &appMetrics } else { envOverride, err = impl.envConfigRepo.Get(environmentProperties.Id) if err != nil { @@ -666,83 +622,3 @@ func (impl PropertiesConfigServiceImpl) CreateEnvironmentPropertiesWithNamespace } return environmentProperties, nil } - -//below method is deprecated - -func (impl PropertiesConfigServiceImpl) EnvMetricsEnableDisable(appMetricRequest *chartService.AppMetricEnableDisableRequest) (*chartService.AppMetricEnableDisableRequest, error) { - // validate app metrics compatibility - var currentChart *chartConfig.EnvConfigOverride - var err error - currentChart, err = impl.envConfigRepo.FindLatestChartForAppByAppIdAndEnvId(appMetricRequest.AppId, appMetricRequest.EnvironmentId) - if err != nil && !errors.IsNotFound(err) { - impl.logger.Error(err) - return nil, err - } - if errors.IsNotFound(err) { - impl.logger.Errorw("no chart configured for this app", "appId", appMetricRequest.AppId) - err = &util.ApiError{ - InternalMessage: "no chart configured for this app", - UserMessage: "no chart configured for this app", - } - return nil, err - } - if appMetricRequest.IsAppMetricsEnabled == true { - chartRefValue, err := impl.chartRefRepository.FindById(currentChart.Chart.ChartRefId) - if err != nil { - impl.logger.Errorw("error in finding ref chart by id", "err", err) - return nil, err - } - if !(chartRefValue.IsAppMetricsSupported) { - err = &util.ApiError{ - InternalMessage: "chart version in not compatible for app metrics", - UserMessage: "chart version in not compatible for app metrics", - } - return nil, err - } - } - // update and create env level app metrics - envLevelAppMetrics, err := impl.envLevelAppMetricsRepository.FindByAppIdAndEnvId(appMetricRequest.AppId, appMetricRequest.EnvironmentId) - if err != nil && !util.IsErrNoRows(err) { - impl.logger.Error("err", err) - return nil, err - } - if envLevelAppMetrics == nil || envLevelAppMetrics.Id == 0 { - infraMetrics := true - envLevelAppMetrics = &repository.EnvLevelAppMetrics{ - AppId: appMetricRequest.AppId, - EnvId: appMetricRequest.EnvironmentId, - AppMetrics: &appMetricRequest.IsAppMetricsEnabled, - InfraMetrics: &infraMetrics, - AuditLog: sql.AuditLog{ - CreatedOn: time.Now(), - UpdatedOn: time.Now(), - CreatedBy: appMetricRequest.UserId, - UpdatedBy: appMetricRequest.UserId, - }, - } - err = impl.envLevelAppMetricsRepository.Save(envLevelAppMetrics) - if err != nil { - impl.logger.Error("err", err) - return nil, err - } - } else { - envLevelAppMetrics.AppMetrics = &appMetricRequest.IsAppMetricsEnabled - envLevelAppMetrics.UpdatedOn = time.Now() - envLevelAppMetrics.UpdatedBy = appMetricRequest.UserId - err = impl.envLevelAppMetricsRepository.Update(envLevelAppMetrics) - if err != nil { - impl.logger.Error("err", err) - return nil, err - } - } - //updating audit log details of chart as history service uses it - currentChart.UpdatedOn = time.Now() - currentChart.UpdatedBy = appMetricRequest.UserId - //creating history entry - err = impl.deploymentTemplateHistoryService.CreateDeploymentTemplateHistoryFromEnvOverrideTemplate(currentChart, nil, appMetricRequest.IsAppMetricsEnabled, 0) - if err != nil { - impl.logger.Errorw("error in creating entry for env deployment template history", "err", err, "envOverride", currentChart) - return nil, err - } - return appMetricRequest, err -} diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index 1391bc5ce6..7a86482794 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -189,7 +189,6 @@ type WorkflowDagExecutorImpl struct { helmAppClient client2.HelmAppClient chartRefRepository chartRepoRepository.ChartRefRepository environmentConfigRepository chartConfig.EnvConfigOverrideRepository - envLevelMetricsRepository repository.EnvLevelAppMetricsRepository dbMigrationConfigRepository pipelineConfig.DbMigrationConfigRepository mergeUtil *util.MergeUtil gitOpsConfigRepository repository.GitOpsConfigRepository @@ -297,7 +296,6 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi helmAppClient client2.HelmAppClient, chartRefRepository chartRepoRepository.ChartRefRepository, environmentConfigRepository chartConfig.EnvConfigOverrideRepository, - envLevelMetricsRepository repository.EnvLevelAppMetricsRepository, dbMigrationConfigRepository pipelineConfig.DbMigrationConfigRepository, mergeUtil *util.MergeUtil, gitOpsConfigRepository repository.GitOpsConfigRepository, @@ -373,7 +371,6 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi helmAppClient: helmAppClient, chartRefRepository: chartRefRepository, environmentConfigRepository: environmentConfigRepository, - envLevelMetricsRepository: envLevelMetricsRepository, dbMigrationConfigRepository: dbMigrationConfigRepository, mergeUtil: mergeUtil, gitOpsConfigRepository: gitOpsConfigRepository, @@ -3809,23 +3806,14 @@ func (impl *WorkflowDagExecutorImpl) GetAppMetricsByTriggerType(overrideRequest appMetrics = deploymentTemplateHistory.IsAppMetricsEnabled } else if overrideRequest.DeploymentWithConfig == bean.DEPLOYMENT_CONFIG_TYPE_LAST_SAVED { - _, span := otel.Tracer("orchestrator").Start(ctx, "appLevelMetricsRepository.FindByAppId") - isAppLevelMetricsEnabled, err := impl.deployedAppMetricsService.GetMetricsFlagByAppIdEvenIfNotInDb(overrideRequest.AppId) + _, span := otel.Tracer("orchestrator").Start(ctx, "deployedAppMetricsService.GetMetricsFlagForAPipelineByAppIdAndEnvId") + isAppMetricsEnabled, err := impl.deployedAppMetricsService.GetMetricsFlagForAPipelineByAppIdAndEnvId(overrideRequest.AppId, overrideRequest.EnvId) if err != nil { - impl.logger.Errorw("error, GetMetricsFlagByAppIdEvenIfNotInDb", "err", err, "appId", overrideRequest.AppId) + impl.logger.Errorw("error, GetMetricsFlagForAPipelineByAppIdAndEnvId", "err", err, "appId", overrideRequest.AppId, "envId", overrideRequest.EnvId) return appMetrics, err } - appMetrics = isAppLevelMetricsEnabled - _, span = otel.Tracer("orchestrator").Start(ctx, "envLevelMetricsRepository.FindByAppIdAndEnvId") - envLevelMetrics, err := impl.envLevelMetricsRepository.FindByAppIdAndEnvId(overrideRequest.AppId, overrideRequest.EnvId) span.End() - if err != nil && !util.IsErrNoRows(err) { - impl.logger.Errorw("err", err) - return appMetrics, &util.ApiError{InternalMessage: "unable to fetch env level metrics flag"} - } - if envLevelMetrics.Id != 0 && envLevelMetrics.AppMetrics != nil { - appMetrics = *envLevelMetrics.AppMetrics - } + appMetrics = isAppMetricsEnabled } return appMetrics, nil } diff --git a/pkg/pipeline/history/DeploymentTemplateHistoryService.go b/pkg/pipeline/history/DeploymentTemplateHistoryService.go index 1d41cf373d..beeae5af84 100644 --- a/pkg/pipeline/history/DeploymentTemplateHistoryService.go +++ b/pkg/pipeline/history/DeploymentTemplateHistoryService.go @@ -5,7 +5,6 @@ import ( "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" "time" - repository2 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/pkg/auth/user" @@ -40,7 +39,6 @@ type DeploymentTemplateHistoryServiceImpl struct { pipelineRepository pipelineConfig.PipelineRepository chartRepository chartRepoRepository.ChartRepository chartRefRepository chartRepoRepository.ChartRefRepository - envLevelAppMetricsRepository repository2.EnvLevelAppMetricsRepository userService user.UserService cdWorkflowRepository pipelineConfig.CdWorkflowRepository scopedVariableManager variables.ScopedVariableManager @@ -48,13 +46,9 @@ type DeploymentTemplateHistoryServiceImpl struct { } func NewDeploymentTemplateHistoryServiceImpl(logger *zap.SugaredLogger, deploymentTemplateHistoryRepository repository.DeploymentTemplateHistoryRepository, - pipelineRepository pipelineConfig.PipelineRepository, - chartRepository chartRepoRepository.ChartRepository, - chartRefRepository chartRepoRepository.ChartRefRepository, - envLevelAppMetricsRepository repository2.EnvLevelAppMetricsRepository, - userService user.UserService, - cdWorkflowRepository pipelineConfig.CdWorkflowRepository, - scopedVariableManager variables.ScopedVariableManager, + pipelineRepository pipelineConfig.PipelineRepository, chartRepository chartRepoRepository.ChartRepository, + chartRefRepository chartRepoRepository.ChartRefRepository, userService user.UserService, + cdWorkflowRepository pipelineConfig.CdWorkflowRepository, scopedVariableManager variables.ScopedVariableManager, deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService) *DeploymentTemplateHistoryServiceImpl { return &DeploymentTemplateHistoryServiceImpl{ logger: logger, @@ -62,7 +56,6 @@ func NewDeploymentTemplateHistoryServiceImpl(logger *zap.SugaredLogger, deployme pipelineRepository: pipelineRepository, chartRepository: chartRepository, chartRefRepository: chartRefRepository, - envLevelAppMetricsRepository: envLevelAppMetricsRepository, userService: userService, cdWorkflowRepository: cdWorkflowRepository, scopedVariableManager: scopedVariableManager, @@ -208,20 +201,10 @@ func (impl DeploymentTemplateHistoryServiceImpl) CreateDeploymentTemplateHistory if len(chartRef.Name) == 0 { chartRef.Name = "Rollout Deployment" } - isAppMetricsEnabled := false - envLevelAppMetrics, err := impl.envLevelAppMetricsRepository.FindByAppIdAndEnvId(pipeline.AppId, pipeline.EnvironmentId) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in getting env level app metrics", "err", err, "appId", pipeline.AppId, "envId", pipeline.EnvironmentId) + isAppMetricsEnabled, err := impl.deployedAppMetricsService.GetMetricsFlagForAPipelineByAppIdAndEnvId(pipeline.AppId, pipeline.EnvironmentId) + if err != nil { + impl.logger.Errorw("error, GetMetricsFlagForAPipelineByAppIdAndEnvId", "err", err, "appId", pipeline.AppId, "envId", pipeline.EnvironmentId) return nil, err - } else if err == pg.ErrNoRows { - isAppLevelMetricsEnabled, err := impl.deployedAppMetricsService.GetMetricsFlagByAppIdEvenIfNotInDb(pipeline.AppId) - if err != nil { - impl.logger.Errorw("error in getting app level app metrics", "err", err, "appId", pipeline.AppId) - return nil, err - } - isAppMetricsEnabled = isAppLevelMetricsEnabled - } else { - isAppMetricsEnabled = *envLevelAppMetrics.AppMetrics } historyModel := &repository.DeploymentTemplateHistory{ AppId: pipeline.AppId, diff --git a/wire_gen.go b/wire_gen.go index 8520b05196..d330b3674e 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -354,14 +354,14 @@ func InitializeApp() (*App, error) { configMapHistoryServiceImpl := history.NewConfigMapHistoryServiceImpl(sugaredLogger, configMapHistoryRepositoryImpl, pipelineRepositoryImpl, configMapRepositoryImpl, userServiceImpl, scopedVariableCMCSManagerImpl) deploymentTemplateHistoryRepositoryImpl := repository6.NewDeploymentTemplateHistoryRepositoryImpl(sugaredLogger, db) chartRefRepositoryImpl := chartRepoRepository.NewChartRefRepositoryImpl(db) - envLevelAppMetricsRepositoryImpl := repository.NewEnvLevelAppMetricsRepositoryImpl(db, sugaredLogger) scopedVariableManagerImpl, err := variables.NewScopedVariableManagerImpl(sugaredLogger, scopedVariableServiceImpl, variableEntityMappingServiceImpl, variableSnapshotHistoryServiceImpl, variableTemplateParserImpl) if err != nil { return nil, err } appLevelMetricsRepositoryImpl := repository.NewAppLevelMetricsRepositoryImpl(db, sugaredLogger) + envLevelAppMetricsRepositoryImpl := repository.NewEnvLevelAppMetricsRepositoryImpl(db, sugaredLogger) deployedAppMetricsServiceImpl := deployedAppMetrics.NewDeployedAppMetricsServiceImpl(sugaredLogger, chartRefRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl) - deploymentTemplateHistoryServiceImpl := history.NewDeploymentTemplateHistoryServiceImpl(sugaredLogger, deploymentTemplateHistoryRepositoryImpl, pipelineRepositoryImpl, chartRepositoryImpl, chartRefRepositoryImpl, envLevelAppMetricsRepositoryImpl, userServiceImpl, cdWorkflowRepositoryImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl) + deploymentTemplateHistoryServiceImpl := history.NewDeploymentTemplateHistoryServiceImpl(sugaredLogger, deploymentTemplateHistoryRepositoryImpl, pipelineRepositoryImpl, chartRepositoryImpl, chartRefRepositoryImpl, userServiceImpl, cdWorkflowRepositoryImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl) chartWorkingDir := _wireChartWorkingDirValue globalEnvVariables, err := util3.GetGlobalEnvVariables() if err != nil { @@ -375,7 +375,7 @@ func InitializeApp() (*App, error) { Logger: sugaredLogger, } repositoryServiceClientImpl := repository9.NewServiceClientImpl(sugaredLogger, argoCDConnectionManagerImpl) - chartServiceImpl := chart.NewChartServiceImpl(chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, appRepositoryImpl, refChartDir, defaultChart, utilMergeUtil, repositoryServiceClientImpl, chartRefRepositoryImpl, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, envLevelAppMetricsRepositoryImpl, httpClient, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl) + chartServiceImpl := chart.NewChartServiceImpl(chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, appRepositoryImpl, refChartDir, defaultChart, utilMergeUtil, repositoryServiceClientImpl, chartRefRepositoryImpl, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, httpClient, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl) devtronSecretConfig, err := util3.GetDevtronSecretName() if err != nil { return nil, err @@ -469,10 +469,10 @@ func InitializeApp() (*App, error) { customTagServiceImpl := pipeline.NewCustomTagService(sugaredLogger, imageTagRepositoryImpl) pluginInputVariableParserImpl := pipeline.NewPluginInputVariableParserImpl(sugaredLogger, dockerRegistryConfigImpl, customTagServiceImpl) pipelineConfigListenerServiceImpl := pipeline.NewPipelineConfigListenerServiceImpl(sugaredLogger) - workflowDagExecutorImpl := pipeline.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, pubSubClientServiceImpl, appServiceImpl, workflowServiceImpl, ciArtifactRepositoryImpl, ciPipelineRepositoryImpl, materialRepositoryImpl, pipelineOverrideRepositoryImpl, userServiceImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, enforcerImpl, enforcerUtilImpl, tokenCache, acdAuthConfig, eventSimpleFactoryImpl, eventRESTClientImpl, cvePolicyRepositoryImpl, imageScanResultRepositoryImpl, appWorkflowRepositoryImpl, prePostCdScriptHistoryServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineServiceImpl, ciTemplateRepositoryImpl, ciWorkflowRepositoryImpl, appLabelRepositoryImpl, clientImpl, pipelineStageServiceImpl, k8sCommonServiceImpl, variableSnapshotHistoryServiceImpl, globalPluginServiceImpl, pluginInputVariableParserImpl, scopedVariableCMCSManagerImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineStrategyHistoryServiceImpl, manifestPushConfigRepositoryImpl, gitOpsManifestPushServiceImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, appCrudOperationServiceImpl, pipelineConfigRepositoryImpl, dockerRegistryIpsConfigServiceImpl, chartRepositoryImpl, chartTemplateServiceImpl, pipelineStrategyHistoryRepositoryImpl, appRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, argoK8sClientImpl, configMapRepositoryImpl, configMapHistoryRepositoryImpl, refChartDir, helmAppServiceImpl, helmAppClientImpl, chartRefRepositoryImpl, envConfigOverrideRepositoryImpl, envLevelAppMetricsRepositoryImpl, dbMigrationConfigRepositoryImpl, mergeUtil, gitOpsConfigRepositoryImpl, gitFactory, applicationServiceClientImpl, argoClientWrapperServiceImpl, pipelineConfigListenerServiceImpl, customTagServiceImpl, acdConfig, deployedAppMetricsServiceImpl) + workflowDagExecutorImpl := pipeline.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, pubSubClientServiceImpl, appServiceImpl, workflowServiceImpl, ciArtifactRepositoryImpl, ciPipelineRepositoryImpl, materialRepositoryImpl, pipelineOverrideRepositoryImpl, userServiceImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, enforcerImpl, enforcerUtilImpl, tokenCache, acdAuthConfig, eventSimpleFactoryImpl, eventRESTClientImpl, cvePolicyRepositoryImpl, imageScanResultRepositoryImpl, appWorkflowRepositoryImpl, prePostCdScriptHistoryServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineServiceImpl, ciTemplateRepositoryImpl, ciWorkflowRepositoryImpl, appLabelRepositoryImpl, clientImpl, pipelineStageServiceImpl, k8sCommonServiceImpl, variableSnapshotHistoryServiceImpl, globalPluginServiceImpl, pluginInputVariableParserImpl, scopedVariableCMCSManagerImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineStrategyHistoryServiceImpl, manifestPushConfigRepositoryImpl, gitOpsManifestPushServiceImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, appCrudOperationServiceImpl, pipelineConfigRepositoryImpl, dockerRegistryIpsConfigServiceImpl, chartRepositoryImpl, chartTemplateServiceImpl, pipelineStrategyHistoryRepositoryImpl, appRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, argoK8sClientImpl, configMapRepositoryImpl, configMapHistoryRepositoryImpl, refChartDir, helmAppServiceImpl, helmAppClientImpl, chartRefRepositoryImpl, envConfigOverrideRepositoryImpl, dbMigrationConfigRepositoryImpl, mergeUtil, gitOpsConfigRepositoryImpl, gitFactory, applicationServiceClientImpl, argoClientWrapperServiceImpl, pipelineConfigListenerServiceImpl, customTagServiceImpl, acdConfig, deployedAppMetricsServiceImpl) deploymentGroupAppRepositoryImpl := repository.NewDeploymentGroupAppRepositoryImpl(sugaredLogger, db) deploymentGroupServiceImpl := deploymentGroup.NewDeploymentGroupServiceImpl(appRepositoryImpl, sugaredLogger, pipelineRepositoryImpl, ciPipelineRepositoryImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, deploymentGroupAppRepositoryImpl, ciArtifactRepositoryImpl, appWorkflowRepositoryImpl, workflowDagExecutorImpl) - deploymentConfigServiceImpl := pipeline.NewDeploymentConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, pipelineRepositoryImpl, envLevelAppMetricsRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, configMapHistoryServiceImpl, chartRefRepositoryImpl, scopedVariableCMCSManagerImpl, deployedAppMetricsServiceImpl) + deploymentConfigServiceImpl := pipeline.NewDeploymentConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, pipelineRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, configMapHistoryServiceImpl, chartRefRepositoryImpl, scopedVariableCMCSManagerImpl, deployedAppMetricsServiceImpl) pipelineTriggerRestHandlerImpl := restHandler.NewPipelineRestHandler(appServiceImpl, userServiceImpl, validate, enforcerImpl, teamServiceImpl, sugaredLogger, enforcerUtilImpl, workflowDagExecutorImpl, deploymentGroupServiceImpl, argoUserServiceImpl, deploymentConfigServiceImpl) sseSSE := sse.NewSSE() pipelineTriggerRouterImpl := router.NewPipelineTriggerRouter(pipelineTriggerRestHandlerImpl, sseSSE) @@ -501,7 +501,7 @@ func InitializeApp() (*App, error) { ciMaterialConfigServiceImpl := pipeline.NewCiMaterialConfigServiceImpl(sugaredLogger, materialRepositoryImpl, ciTemplateServiceImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, gitMaterialHistoryServiceImpl, pipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl) imageTaggingRepositoryImpl := repository13.NewImageTaggingRepositoryImpl(db) imageTaggingServiceImpl := pipeline.NewImageTaggingServiceImpl(imageTaggingRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, sugaredLogger) - propertiesConfigServiceImpl := pipeline.NewPropertiesConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, chartRefRepositoryImpl, utilMergeUtil, environmentRepositoryImpl, ciCdPipelineOrchestratorImpl, applicationServiceClientImpl, envLevelAppMetricsRepositoryImpl, appLevelMetricsRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl) + propertiesConfigServiceImpl := pipeline.NewPropertiesConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, chartRefRepositoryImpl, utilMergeUtil, environmentRepositoryImpl, ciCdPipelineOrchestratorImpl, applicationServiceClientImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl) pipelineDeploymentServiceTypeConfig, err := pipeline.GetDeploymentServiceTypeConfig() if err != nil { return nil, err @@ -525,7 +525,7 @@ func InitializeApp() (*App, error) { gitRegistryConfigImpl := pipeline.NewGitRegistryConfigImpl(sugaredLogger, gitProviderRepositoryImpl, clientImpl) appListingViewBuilderImpl := app2.NewAppListingViewBuilderImpl(sugaredLogger) linkoutsRepositoryImpl := repository.NewLinkoutsRepositoryImpl(sugaredLogger, db) - appListingServiceImpl := app2.NewAppListingServiceImpl(sugaredLogger, appListingRepositoryImpl, applicationServiceClientImpl, appRepositoryImpl, appListingViewBuilderImpl, pipelineRepositoryImpl, linkoutsRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, cdWorkflowRepositoryImpl, pipelineOverrideRepositoryImpl, environmentRepositoryImpl, argoUserServiceImpl, envConfigOverrideRepositoryImpl, chartRepositoryImpl, ciPipelineRepositoryImpl, dockerRegistryIpsConfigServiceImpl, userRepositoryImpl) + appListingServiceImpl := app2.NewAppListingServiceImpl(sugaredLogger, appListingRepositoryImpl, applicationServiceClientImpl, appRepositoryImpl, appListingViewBuilderImpl, pipelineRepositoryImpl, linkoutsRepositoryImpl, appLevelMetricsRepositoryImpl, cdWorkflowRepositoryImpl, pipelineOverrideRepositoryImpl, environmentRepositoryImpl, argoUserServiceImpl, envConfigOverrideRepositoryImpl, chartRepositoryImpl, ciPipelineRepositoryImpl, dockerRegistryIpsConfigServiceImpl, userRepositoryImpl) deploymentEventHandlerImpl := app2.NewDeploymentEventHandlerImpl(sugaredLogger, appListingServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl) cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, helmAppServiceImpl, pipelineOverrideRepositoryImpl, workflowDagExecutorImpl, appListingServiceImpl, appListingRepositoryImpl, pipelineStatusTimelineRepositoryImpl, applicationServiceClientImpl, argoUserServiceImpl, deploymentEventHandlerImpl, eventRESTClientImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceImpl, appStatusServiceImpl, enforcerUtilImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, appRepositoryImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sUtil, workflowServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl, customTagServiceImpl, argoClientWrapperServiceImpl, appServiceConfig, acdConfig) appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, appRepositoryImpl, userAuthServiceImpl) @@ -535,7 +535,7 @@ func InitializeApp() (*App, error) { imageScanObjectMetaRepositoryImpl := security.NewImageScanObjectMetaRepositoryImpl(db, sugaredLogger) cveStoreRepositoryImpl := security.NewCveStoreRepositoryImpl(db, sugaredLogger) policyServiceImpl := security2.NewPolicyServiceImpl(environmentServiceImpl, sugaredLogger, appRepositoryImpl, pipelineOverrideRepositoryImpl, cvePolicyRepositoryImpl, clusterServiceImplExtended, pipelineRepositoryImpl, imageScanResultRepositoryImpl, imageScanDeployInfoRepositoryImpl, imageScanObjectMetaRepositoryImpl, httpClient, ciArtifactRepositoryImpl, ciCdConfig, imageScanHistoryRepositoryImpl, cveStoreRepositoryImpl, ciTemplateRepositoryImpl) - pipelineConfigRestHandlerImpl := app3.NewPipelineRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, chartServiceImpl, propertiesConfigServiceImpl, dbMigrationServiceImpl, applicationServiceClientImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, environmentServiceImpl, gitRegistryConfigImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, deploymentTemplateServiceImpl, appWorkflowServiceImpl, materialRepositoryImpl, policyServiceImpl, imageScanResultRepositoryImpl, gitProviderRepositoryImpl, argoUserServiceImpl, ciPipelineMaterialRepositoryImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl) + pipelineConfigRestHandlerImpl := app3.NewPipelineRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, chartServiceImpl, propertiesConfigServiceImpl, dbMigrationServiceImpl, applicationServiceClientImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, environmentServiceImpl, gitRegistryConfigImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, deploymentTemplateServiceImpl, appWorkflowServiceImpl, materialRepositoryImpl, policyServiceImpl, imageScanResultRepositoryImpl, gitProviderRepositoryImpl, argoUserServiceImpl, ciPipelineMaterialRepositoryImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, deployedAppMetricsServiceImpl) appWorkflowRestHandlerImpl := restHandler.NewAppWorkflowRestHandlerImpl(sugaredLogger, userServiceImpl, appWorkflowServiceImpl, teamServiceImpl, enforcerImpl, pipelineBuilderImpl, appRepositoryImpl, enforcerUtilImpl) webhookEventDataRepositoryImpl := repository.NewWebhookEventDataRepositoryImpl(db) webhookEventDataConfigImpl := pipeline.NewWebhookEventDataConfigImpl(sugaredLogger, webhookEventDataRepositoryImpl) @@ -733,7 +733,7 @@ func InitializeApp() (*App, error) { telemetryRestHandlerImpl := restHandler.NewTelemetryRestHandlerImpl(sugaredLogger, telemetryEventClientImplExtended, enforcerImpl, userServiceImpl) telemetryRouterImpl := router.NewTelemetryRouterImpl(sugaredLogger, telemetryRestHandlerImpl) bulkUpdateRepositoryImpl := bulkUpdate.NewBulkUpdateRepository(db, sugaredLogger) - bulkUpdateServiceImpl, err := bulkAction.NewBulkUpdateServiceImpl(bulkUpdateRepositoryImpl, chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, defaultChart, utilMergeUtil, repositoryServiceClientImpl, chartRefRepositoryImpl, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, httpClient, appRepositoryImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, workflowDagExecutorImpl, cdWorkflowRepositoryImpl, pipelineBuilderImpl, helmAppServiceImpl, enforcerUtilImpl, enforcerUtilHelmImpl, ciHandlerImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, appWorkflowServiceImpl, pubSubClientServiceImpl, argoUserServiceImpl, scopedVariableManagerImpl) + bulkUpdateServiceImpl, err := bulkAction.NewBulkUpdateServiceImpl(bulkUpdateRepositoryImpl, chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, defaultChart, utilMergeUtil, repositoryServiceClientImpl, chartRefRepositoryImpl, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, httpClient, appRepositoryImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, workflowDagExecutorImpl, cdWorkflowRepositoryImpl, pipelineBuilderImpl, helmAppServiceImpl, enforcerUtilImpl, enforcerUtilHelmImpl, ciHandlerImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, appWorkflowServiceImpl, pubSubClientServiceImpl, argoUserServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl) if err != nil { return nil, err } From c7cfc9e11cffae54f4b4aa91f950884c5810ce52 Mon Sep 17 00:00:00 2001 From: Nishant <58689354+nishant-d@users.noreply.github.com> Date: Mon, 15 Jan 2024 16:52:42 +0530 Subject: [PATCH 04/64] chore: Removed unused jira and migration integration (#4498) * removed unsued jira integration * removed test-suite-code * db migration conf removal * chore: removed unused injection * chore: removed dead code * added: migration script --------- Co-authored-by: Ash-exp --- Wire.go | 42 +-- .../chartProvider/ChartProviderRestHandler.go | 6 +- .../AppStoreDeploymentRestHandler.go | 6 +- .../deployment/CommonDeploymentRestHandler.go | 6 +- api/deployment/DeploymentConfigRestHandler.go | 6 +- .../application/k8sApplicationRestHandler.go | 6 +- api/restHandler/BulkUpdateRestHandler.go | 3 - api/restHandler/ChartGroupRestHandler.go | 6 +- api/restHandler/CoreAppRestHandler.go | 6 +- api/restHandler/DeploymentGroupRestHandler.go | 6 +- api/restHandler/GitHostRestHandler.go | 6 +- api/restHandler/GitProviderRestHandler.go | 8 +- api/restHandler/GlobalCMCSRestHandler.go | 6 +- api/restHandler/ImageScanRestHandler.go | 6 +- api/restHandler/JiraRestHandler.go | 111 ------- api/restHandler/MigrateDbRestHandler.go | 217 -------------- api/restHandler/NotificationRestHandler.go | 8 +- api/restHandler/PolicyRestHandler.go | 8 +- api/restHandler/TelemetryRestHandler.go | 4 +- api/restHandler/TestSuitRestHandler.go | 276 ------------------ .../app/DeploymentPipelineRestHandler.go | 131 --------- .../app/PipelineConfigRestHandler.go | 3 - api/router/MigrateDbRouter.go | 51 ---- api/router/PipelineConfigRouter.go | 4 - api/router/ProjectManagementRouter.go | 45 --- api/router/TestSuites.go | 45 --- api/router/router.go | 21 +- api/terminal/UserTerminalAccessRestHandler.go | 6 +- client/events/EventBuilder.go | 6 +- client/events/EventClient.go | 19 -- internal/sql/repository/DbConfigRepository.go | 100 ------- .../sql/repository/JiraAccountRepository.go | 92 ------ .../pipelineConfig/DbMigrationConfig.go | 85 ------ pkg/app/AppService.go | 9 +- .../deployment/service/InstalledAppService.go | 6 +- .../service/InstalledAppService_test.go | 6 +- .../gitops/AppStoreDeploymentArgoCdService.go | 6 +- pkg/auth/user/bean/bean.go | 36 +-- .../user/repository/UserAuditRepository.go | 2 +- pkg/auth/user/repository/UserRepository.go | 4 +- pkg/jira/JiraService.go | 174 ----------- pkg/notifier/NotificationConfigService.go | 6 +- pkg/pipeline/AppArtifactManager.go | 6 +- pkg/pipeline/CiHandler.go | 75 ----- pkg/pipeline/DbConfigService.go | 153 ---------- pkg/pipeline/DbMigrationService.go | 119 -------- pkg/pipeline/WorkflowDagExecutor.go | 234 +-------------- .../jira/JiraAccountValidator.go | 56 ---- .../jira/JiraManagementService.go | 98 ------- pkg/security/ImageScanService.go | 6 +- scripts/sql/210_unused_integration.up.sql | 12 + scripts/sql/210_unused_intergration.down.sql | 125 ++++++++ wire_gen.go | 32 +- 53 files changed, 247 insertions(+), 2269 deletions(-) delete mode 100644 api/restHandler/JiraRestHandler.go delete mode 100644 api/restHandler/MigrateDbRestHandler.go delete mode 100644 api/restHandler/TestSuitRestHandler.go delete mode 100644 api/router/MigrateDbRouter.go delete mode 100644 api/router/ProjectManagementRouter.go delete mode 100644 api/router/TestSuites.go delete mode 100644 internal/sql/repository/DbConfigRepository.go delete mode 100644 internal/sql/repository/JiraAccountRepository.go delete mode 100644 internal/sql/repository/pipelineConfig/DbMigrationConfig.go delete mode 100644 pkg/jira/JiraService.go delete mode 100644 pkg/pipeline/DbConfigService.go delete mode 100644 pkg/pipeline/DbMigrationService.go delete mode 100644 pkg/projectManagementService/jira/JiraAccountValidator.go delete mode 100644 pkg/projectManagementService/jira/JiraManagementService.go create mode 100644 scripts/sql/210_unused_integration.up.sql create mode 100644 scripts/sql/210_unused_intergration.down.sql diff --git a/Wire.go b/Wire.go index 9cc5e42f2f..69890f5668 100644 --- a/Wire.go +++ b/Wire.go @@ -63,7 +63,6 @@ import ( eClient "github.com/devtron-labs/devtron/client/events" "github.com/devtron-labs/devtron/client/gitSensor" "github.com/devtron-labs/devtron/client/grafana" - jClient "github.com/devtron-labs/devtron/client/jira" "github.com/devtron-labs/devtron/client/lens" "github.com/devtron-labs/devtron/client/telemetry" "github.com/devtron-labs/devtron/internal/sql/repository" @@ -104,7 +103,6 @@ import ( "github.com/devtron-labs/devtron/pkg/generateManifest" "github.com/devtron-labs/devtron/pkg/git" "github.com/devtron-labs/devtron/pkg/gitops" - jira2 "github.com/devtron-labs/devtron/pkg/jira" "github.com/devtron-labs/devtron/pkg/kubernetesResourceAuditLogs" repository7 "github.com/devtron-labs/devtron/pkg/kubernetesResourceAuditLogs/repository" "github.com/devtron-labs/devtron/pkg/notifier" @@ -116,7 +114,6 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/plugin" repository6 "github.com/devtron-labs/devtron/pkg/plugin/repository" - "github.com/devtron-labs/devtron/pkg/projectManagementService/jira" resourceGroup2 "github.com/devtron-labs/devtron/pkg/resourceGroup" "github.com/devtron-labs/devtron/pkg/resourceQualifiers" "github.com/devtron-labs/devtron/pkg/security" @@ -258,10 +255,7 @@ func InitializeApp() (*App, error) { wire.Bind(new(pipeline.CiCdPipelineOrchestrator), new(*pipeline.CiCdPipelineOrchestratorImpl)), pipelineConfig.NewMaterialRepositoryImpl, wire.Bind(new(pipelineConfig.MaterialRepository), new(*pipelineConfig.MaterialRepositoryImpl)), - router.NewMigrateDbRouterImpl, - wire.Bind(new(router.MigrateDbRouter), new(*router.MigrateDbRouterImpl)), - restHandler.NewMigrateDbRestHandlerImpl, - wire.Bind(new(restHandler.MigrateDbRestHandler), new(*restHandler.MigrateDbRestHandlerImpl)), + util.NewChartTemplateServiceImpl, wire.Bind(new(util.ChartTemplateService), new(*util.ChartTemplateServiceImpl)), util.NewChartDeploymentServiceImpl, @@ -329,23 +323,8 @@ func InitializeApp() (*App, error) { pipeline.NewPropertiesConfigServiceImpl, wire.Bind(new(pipeline.PropertiesConfigService), new(*pipeline.PropertiesConfigServiceImpl)), - router.NewProjectManagementRouterImpl, - wire.Bind(new(router.ProjectManagementRouter), new(*router.ProjectManagementRouterImpl)), - - restHandler.NewJiraRestHandlerImpl, - wire.Bind(new(restHandler.JiraRestHandler), new(*restHandler.JiraRestHandlerImpl)), - - jira2.NewProjectManagementServiceImpl, - wire.Bind(new(jira2.ProjectManagementService), new(*jira2.ProjectManagementServiceImpl)), - - jira.NewAccountServiceImpl, - wire.Bind(new(jira.AccountService), new(*jira.AccountServiceImpl)), - util.NewHttpClient, - jClient.NewJiraClientImpl, - wire.Bind(new(jClient.JiraClient), new(*jClient.JiraClientImpl)), - eClient.NewEventRESTClientImpl, wire.Bind(new(eClient.EventClient), new(*eClient.EventRESTClientImpl)), @@ -354,11 +333,6 @@ func InitializeApp() (*App, error) { eClient.NewEventSimpleFactoryImpl, wire.Bind(new(eClient.EventFactory), new(*eClient.EventSimpleFactoryImpl)), - repository.NewJiraAccountRepositoryImpl, - wire.Bind(new(repository.JiraAccountRepository), new(*repository.JiraAccountRepositoryImpl)), - jira.NewAccountValidatorImpl, - wire.Bind(new(jira.AccountValidator), new(*jira.AccountValidatorImpl)), - repository.NewCiArtifactRepositoryImpl, wire.Bind(new(repository.CiArtifactRepository), new(*repository.CiArtifactRepositoryImpl)), pipeline.NewWebhookServiceImpl, @@ -402,15 +376,6 @@ func InitializeApp() (*App, error) { //ArgoUtil.NewRepositoryService, //wire.Bind(new(ArgoUtil.RepositoryService), new(ArgoUtil.RepositoryServiceImpl)), - pipelineConfig.NewDbMigrationConfigRepositoryImpl, - wire.Bind(new(pipelineConfig.DbMigrationConfigRepository), new(*pipelineConfig.DbMigrationConfigRepositoryImpl)), - pipeline.NewDbConfigService, - wire.Bind(new(pipeline.DbConfigService), new(*pipeline.DbConfigServiceImpl)), - - repository.NewDbConfigRepositoryImpl, - wire.Bind(new(repository.DbConfigRepository), new(*repository.DbConfigRepositoryImpl)), - pipeline.NewDbMogrationService, - wire.Bind(new(pipeline.DbMigrationService), new(*pipeline.DbMigrationServiceImpl)), //ArgoUtil.NewClusterServiceImpl, //wire.Bind(new(ArgoUtil.ClusterService), new(ArgoUtil.ClusterServiceImpl)), pipeline.GetEcrConfig, @@ -635,11 +600,6 @@ func InitializeApp() (*App, error) { commonService.NewCommonServiceImpl, wire.Bind(new(commonService.CommonService), new(*commonService.CommonServiceImpl)), - router.NewTestSuitRouterImpl, - wire.Bind(new(router.TestSuitRouter), new(*router.TestSuitRouterImpl)), - restHandler.NewTestSuitRestHandlerImpl, - wire.Bind(new(restHandler.TestSuitRestHandler), new(*restHandler.TestSuitRestHandlerImpl)), - router.NewImageScanRouterImpl, wire.Bind(new(router.ImageScanRouter), new(*router.ImageScanRouterImpl)), restHandler.NewImageScanRestHandlerImpl, diff --git a/api/appStore/chartProvider/ChartProviderRestHandler.go b/api/appStore/chartProvider/ChartProviderRestHandler.go index 797a361648..6fd199dbf0 100644 --- a/api/appStore/chartProvider/ChartProviderRestHandler.go +++ b/api/appStore/chartProvider/ChartProviderRestHandler.go @@ -39,9 +39,9 @@ type ChartProviderRestHandler interface { type ChartProviderRestHandlerImpl struct { Logger *zap.SugaredLogger chartProviderService chartProvider.ChartProviderService - validator *validator.Validate - userAuthService user.UserService - enforcer casbin.Enforcer + validator *validator.Validate + userAuthService user.UserService + enforcer casbin.Enforcer } func NewChartProviderRestHandlerImpl(Logger *zap.SugaredLogger, userAuthService user.UserService, validator *validator.Validate, chartProviderService chartProvider.ChartProviderService, diff --git a/api/appStore/deployment/AppStoreDeploymentRestHandler.go b/api/appStore/deployment/AppStoreDeploymentRestHandler.go index 3d6d9199bf..cffb21ff55 100644 --- a/api/appStore/deployment/AppStoreDeploymentRestHandler.go +++ b/api/appStore/deployment/AppStoreDeploymentRestHandler.go @@ -59,9 +59,9 @@ type AppStoreDeploymentRestHandler interface { } type AppStoreDeploymentRestHandlerImpl struct { - Logger *zap.SugaredLogger - userAuthService user.UserService - enforcer casbin.Enforcer + Logger *zap.SugaredLogger + userAuthService user.UserService + enforcer casbin.Enforcer enforcerUtil rbac.EnforcerUtil enforcerUtilHelm rbac.EnforcerUtilHelm appStoreDeploymentService service.AppStoreDeploymentService diff --git a/api/appStore/deployment/CommonDeploymentRestHandler.go b/api/appStore/deployment/CommonDeploymentRestHandler.go index ffc200419f..d7bfac9fd8 100644 --- a/api/appStore/deployment/CommonDeploymentRestHandler.go +++ b/api/appStore/deployment/CommonDeploymentRestHandler.go @@ -51,9 +51,9 @@ type CommonDeploymentRestHandler interface { } type CommonDeploymentRestHandlerImpl struct { - Logger *zap.SugaredLogger - userAuthService user.UserService - enforcer casbin.Enforcer + Logger *zap.SugaredLogger + userAuthService user.UserService + enforcer casbin.Enforcer enforcerUtil rbac.EnforcerUtil enforcerUtilHelm rbac.EnforcerUtilHelm appStoreDeploymentService service.AppStoreDeploymentService diff --git a/api/deployment/DeploymentConfigRestHandler.go b/api/deployment/DeploymentConfigRestHandler.go index 53be9a4b6a..9e17ce9f97 100644 --- a/api/deployment/DeploymentConfigRestHandler.go +++ b/api/deployment/DeploymentConfigRestHandler.go @@ -31,9 +31,9 @@ type DeploymentConfigRestHandler interface { } type DeploymentConfigRestHandlerImpl struct { - Logger *zap.SugaredLogger - userAuthService user.UserService - enforcer casbin.Enforcer + Logger *zap.SugaredLogger + userAuthService user.UserService + enforcer casbin.Enforcer validator *validator.Validate refChartDir chartRepoRepository.RefChartDir chartService chart.ChartService diff --git a/api/k8s/application/k8sApplicationRestHandler.go b/api/k8s/application/k8sApplicationRestHandler.go index c89444ec5b..0fa90296ab 100644 --- a/api/k8s/application/k8sApplicationRestHandler.go +++ b/api/k8s/application/k8sApplicationRestHandler.go @@ -62,9 +62,9 @@ type K8sApplicationRestHandlerImpl struct { validator *validator.Validate enforcerUtil rbac.EnforcerUtil enforcerUtilHelm rbac.EnforcerUtilHelm - helmAppService client.HelmAppService - userService user.UserService - k8sCommonService k8s.K8sCommonService + helmAppService client.HelmAppService + userService user.UserService + k8sCommonService k8s.K8sCommonService } func NewK8sApplicationRestHandlerImpl(logger *zap.SugaredLogger, k8sApplicationService application2.K8sApplicationService, pump connector.Pump, terminalSessionHandler terminal.TerminalSessionHandler, enforcer casbin.Enforcer, enforcerUtilHelm rbac.EnforcerUtilHelm, enforcerUtil rbac.EnforcerUtil, helmAppService client.HelmAppService, userService user.UserService, k8sCommonService k8s.K8sCommonService, validator *validator.Validate) *K8sApplicationRestHandlerImpl { diff --git a/api/restHandler/BulkUpdateRestHandler.go b/api/restHandler/BulkUpdateRestHandler.go index da3b595c81..3a4a0293a6 100644 --- a/api/restHandler/BulkUpdateRestHandler.go +++ b/api/restHandler/BulkUpdateRestHandler.go @@ -51,7 +51,6 @@ type BulkUpdateRestHandlerImpl struct { bulkUpdateService bulkAction.BulkUpdateService chartService chart.ChartService propertiesConfigService pipeline.PropertiesConfigService - dbMigrationService pipeline.DbMigrationService application application.ServiceClient userAuthService user.UserService validator *validator.Validate @@ -76,7 +75,6 @@ func NewBulkUpdateRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, logg bulkUpdateService bulkAction.BulkUpdateService, chartService chart.ChartService, propertiesConfigService pipeline.PropertiesConfigService, - dbMigrationService pipeline.DbMigrationService, application application.ServiceClient, userAuthService user.UserService, teamService team.TeamService, @@ -99,7 +97,6 @@ func NewBulkUpdateRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, logg bulkUpdateService: bulkUpdateService, chartService: chartService, propertiesConfigService: propertiesConfigService, - dbMigrationService: dbMigrationService, application: application, userAuthService: userAuthService, validator: validator, diff --git a/api/restHandler/ChartGroupRestHandler.go b/api/restHandler/ChartGroupRestHandler.go index 46746277f3..9a0b2dc03e 100644 --- a/api/restHandler/ChartGroupRestHandler.go +++ b/api/restHandler/ChartGroupRestHandler.go @@ -36,9 +36,9 @@ const CHART_GROUP_DELETE_SUCCESS_RESP = "Chart group deleted successfully." type ChartGroupRestHandlerImpl struct { ChartGroupService service.ChartGroupService - Logger *zap.SugaredLogger - userAuthService user.UserService - enforcer casbin.Enforcer + Logger *zap.SugaredLogger + userAuthService user.UserService + enforcer casbin.Enforcer validator *validator.Validate } diff --git a/api/restHandler/CoreAppRestHandler.go b/api/restHandler/CoreAppRestHandler.go index 20566db8c4..5cd951d7d8 100644 --- a/api/restHandler/CoreAppRestHandler.go +++ b/api/restHandler/CoreAppRestHandler.go @@ -75,9 +75,9 @@ type CoreAppRestHandler interface { } type CoreAppRestHandlerImpl struct { - logger *zap.SugaredLogger - userAuthService user.UserService - validator *validator.Validate + logger *zap.SugaredLogger + userAuthService user.UserService + validator *validator.Validate enforcerUtil rbac.EnforcerUtil enforcer casbin.Enforcer appCrudOperationService app.AppCrudOperationService diff --git a/api/restHandler/DeploymentGroupRestHandler.go b/api/restHandler/DeploymentGroupRestHandler.go index 399ca23972..d8889d6e97 100644 --- a/api/restHandler/DeploymentGroupRestHandler.go +++ b/api/restHandler/DeploymentGroupRestHandler.go @@ -51,9 +51,9 @@ type DeploymentGroupRestHandlerImpl struct { logger *zap.SugaredLogger validator *validator.Validate enforcer casbin.Enforcer - teamService team.TeamService - userAuthService user.UserService - enforcerUtil rbac.EnforcerUtil + teamService team.TeamService + userAuthService user.UserService + enforcerUtil rbac.EnforcerUtil } func NewDeploymentGroupRestHandlerImpl(deploymentGroupService deploymentGroup.DeploymentGroupService, logger *zap.SugaredLogger, diff --git a/api/restHandler/GitHostRestHandler.go b/api/restHandler/GitHostRestHandler.go index d96e5f20c1..1048ff4a5f 100644 --- a/api/restHandler/GitHostRestHandler.go +++ b/api/restHandler/GitHostRestHandler.go @@ -45,9 +45,9 @@ type GitHostRestHandler interface { type GitHostRestHandlerImpl struct { logger *zap.SugaredLogger - gitHostConfig pipeline.GitHostConfig - userAuthService user.UserService - validator *validator.Validate + gitHostConfig pipeline.GitHostConfig + userAuthService user.UserService + validator *validator.Validate enforcer casbin.Enforcer gitSensorClient gitSensor.Client gitProviderConfig pipeline.GitRegistryConfig diff --git a/api/restHandler/GitProviderRestHandler.go b/api/restHandler/GitProviderRestHandler.go index 0a0e94ceaa..3c0b80d451 100644 --- a/api/restHandler/GitProviderRestHandler.go +++ b/api/restHandler/GitProviderRestHandler.go @@ -48,9 +48,8 @@ type GitProviderRestHandlerImpl struct { dockerRegistryConfig pipeline.DockerRegistryConfig logger *zap.SugaredLogger gitRegistryConfig pipeline.GitRegistryConfig - dbConfigService pipeline.DbConfigService - userAuthService user.UserService - validator *validator.Validate + userAuthService user.UserService + validator *validator.Validate enforcer casbin.Enforcer teamService team.TeamService deleteServiceFullMode delete2.DeleteServiceFullMode @@ -59,14 +58,13 @@ type GitProviderRestHandlerImpl struct { func NewGitProviderRestHandlerImpl(dockerRegistryConfig pipeline.DockerRegistryConfig, logger *zap.SugaredLogger, gitRegistryConfig pipeline.GitRegistryConfig, - dbConfigService pipeline.DbConfigService, userAuthService user.UserService, + userAuthService user.UserService, validator *validator.Validate, enforcer casbin.Enforcer, teamService team.TeamService, deleteServiceFullMode delete2.DeleteServiceFullMode) *GitProviderRestHandlerImpl { return &GitProviderRestHandlerImpl{ dockerRegistryConfig: dockerRegistryConfig, logger: logger, gitRegistryConfig: gitRegistryConfig, - dbConfigService: dbConfigService, userAuthService: userAuthService, validator: validator, enforcer: enforcer, diff --git a/api/restHandler/GlobalCMCSRestHandler.go b/api/restHandler/GlobalCMCSRestHandler.go index b1e294ce7e..4d6f32f36a 100644 --- a/api/restHandler/GlobalCMCSRestHandler.go +++ b/api/restHandler/GlobalCMCSRestHandler.go @@ -24,9 +24,9 @@ type GlobalCMCSRestHandler interface { } type GlobalCMCSRestHandlerImpl struct { - logger *zap.SugaredLogger - userAuthService user.UserService - validator *validator.Validate + logger *zap.SugaredLogger + userAuthService user.UserService + validator *validator.Validate enforcer casbin.Enforcer globalCMCSService pipeline.GlobalCMCSService } diff --git a/api/restHandler/ImageScanRestHandler.go b/api/restHandler/ImageScanRestHandler.go index 12222a7bc5..ca33c82b34 100644 --- a/api/restHandler/ImageScanRestHandler.go +++ b/api/restHandler/ImageScanRestHandler.go @@ -43,9 +43,9 @@ type ImageScanRestHandler interface { type ImageScanRestHandlerImpl struct { logger *zap.SugaredLogger - imageScanService security.ImageScanService - userService user.UserService - enforcer casbin.Enforcer + imageScanService security.ImageScanService + userService user.UserService + enforcer casbin.Enforcer enforcerUtil rbac.EnforcerUtil environmentService cluster.EnvironmentService } diff --git a/api/restHandler/JiraRestHandler.go b/api/restHandler/JiraRestHandler.go deleted file mode 100644 index 117cd1dac0..0000000000 --- a/api/restHandler/JiraRestHandler.go +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright (c) 2020 Devtron Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package restHandler - -import ( - "encoding/json" - "net/http" - - "github.com/devtron-labs/devtron/api/restHandler/common" - "github.com/devtron-labs/devtron/pkg/auth/user" - "github.com/devtron-labs/devtron/pkg/jira" - "go.uber.org/zap" - "gopkg.in/go-playground/validator.v9" -) - -type JiraRestHandler interface { - SaveAccountConfig(w http.ResponseWriter, r *http.Request) - UpdateIssueStatus(w http.ResponseWriter, r *http.Request) -} - -type JiraRestHandlerImpl struct { - jiraService jira.ProjectManagementService - logger *zap.SugaredLogger - userAuthService user.UserService - validator *validator.Validate -} - -func NewJiraRestHandlerImpl(jiraService jira.ProjectManagementService, logger *zap.SugaredLogger, userAuthService user.UserService, validator *validator.Validate) *JiraRestHandlerImpl { - return &JiraRestHandlerImpl{ - jiraService: jiraService, - logger: logger, - userAuthService: userAuthService, - validator: validator, - } -} - -func (impl JiraRestHandlerImpl) SaveAccountConfig(w http.ResponseWriter, r *http.Request) { - decoder := json.NewDecoder(r.Body) - userId, err := impl.userAuthService.GetLoggedInUser(r) - if userId == 0 || err != nil { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return - } - var jiraConfigBean jira.ConfigBean - err = decoder.Decode(&jiraConfigBean) - if err != nil { - impl.logger.Errorw("request err, SaveAccountConfig", "err", err, "payload", jiraConfigBean) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - impl.logger.Infow("request payload, SaveAccountConfig", "err", err, "payload", jiraConfigBean) - err = impl.validator.Struct(jiraConfigBean) - if err != nil { - impl.logger.Errorw("validation err, SaveAccountConfig", "err", err, "payload", jiraConfigBean) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - - account, err := impl.jiraService.SaveAccountDetails(&jiraConfigBean, userId) - if err != nil { - impl.logger.Errorw("service err, SaveAccountConfig", "err", err, "payload", jiraConfigBean) - common.WriteJsonResp(w, err, "error in saving jira config", http.StatusInternalServerError) - return - } - common.WriteJsonResp(w, err, account.Id, http.StatusOK) -} - -func (impl JiraRestHandlerImpl) UpdateIssueStatus(w http.ResponseWriter, r *http.Request) { - userId, err := impl.userAuthService.GetLoggedInUser(r) - if userId == 0 || err != nil { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return - } - var updateBean jira.UpdateIssueBean - err = json.NewDecoder(r.Body).Decode(&updateBean) - if err != nil { - impl.logger.Errorw("request err, UpdateIssueStatus", "err", err, "payload", updateBean) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - impl.logger.Infow("request payload, UpdateIssueStatus", "err", err, "payload", updateBean) - err = impl.validator.Struct(updateBean) - if err != nil { - impl.logger.Errorw("validation err, UpdateIssueStatus", "err", err, "payload", updateBean) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - res, err := impl.jiraService.UpdateJiraStatus(&updateBean, userId) - if err != nil { - impl.logger.Errorw("service err, UpdateIssueStatus", "err", err, "payload", updateBean) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - w.Header().Set("Content-Type", "application/json") - common.WriteJsonResp(w, err, res, http.StatusOK) -} diff --git a/api/restHandler/MigrateDbRestHandler.go b/api/restHandler/MigrateDbRestHandler.go deleted file mode 100644 index aef4fe4beb..0000000000 --- a/api/restHandler/MigrateDbRestHandler.go +++ /dev/null @@ -1,217 +0,0 @@ -/* - * Copyright (c) 2020 Devtron Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package restHandler - -import ( - "encoding/json" - "net/http" - "strconv" - - "github.com/devtron-labs/devtron/api/restHandler/common" - "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" - "github.com/devtron-labs/devtron/pkg/auth/user" - "github.com/devtron-labs/devtron/pkg/pipeline" - "github.com/devtron-labs/devtron/pkg/pipeline/types" - "github.com/gorilla/mux" - "go.uber.org/zap" - "gopkg.in/go-playground/validator.v9" -) - -type MigrateDbRestHandler interface { - SaveDbConfig(w http.ResponseWriter, r *http.Request) - FetchAllDbConfig(w http.ResponseWriter, r *http.Request) - FetchOneDbConfig(w http.ResponseWriter, r *http.Request) - UpdateDbConfig(w http.ResponseWriter, r *http.Request) - FetchDbConfigForAutoComp(w http.ResponseWriter, r *http.Request) -} -type MigrateDbRestHandlerImpl struct { - dockerRegistryConfig pipeline.DockerRegistryConfig - logger *zap.SugaredLogger - gitRegistryConfig pipeline.GitRegistryConfig - dbConfigService pipeline.DbConfigService - userAuthService user.UserService - validator *validator.Validate - dbMigrationService pipeline.DbMigrationService - enforcer casbin.Enforcer -} - -func NewMigrateDbRestHandlerImpl(dockerRegistryConfig pipeline.DockerRegistryConfig, - logger *zap.SugaredLogger, gitRegistryConfig pipeline.GitRegistryConfig, - dbConfigService pipeline.DbConfigService, userAuthService user.UserService, - validator *validator.Validate, dbMigrationService pipeline.DbMigrationService, - enforcer casbin.Enforcer) *MigrateDbRestHandlerImpl { - return &MigrateDbRestHandlerImpl{ - dockerRegistryConfig: dockerRegistryConfig, - logger: logger, - gitRegistryConfig: gitRegistryConfig, - dbConfigService: dbConfigService, - userAuthService: userAuthService, - validator: validator, - dbMigrationService: dbMigrationService, - enforcer: enforcer, - } -} - -func (impl MigrateDbRestHandlerImpl) SaveDbConfig(w http.ResponseWriter, r *http.Request) { - decoder := json.NewDecoder(r.Body) - userId, err := impl.userAuthService.GetLoggedInUser(r) - if userId == 0 || err != nil { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return - } - var bean types.DbConfigBean - err = decoder.Decode(&bean) - if err != nil { - impl.logger.Errorw("request err, SaveDbConfig", "err", err, "payload", bean) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - bean.UserId = userId - impl.logger.Errorw("request payload, SaveDbConfig", "err", err, "payload", bean) - err = impl.validator.Struct(bean) - if err != nil { - impl.logger.Errorw("validation err, SaveDbConfig", "err", err, "payload", bean) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - - // RBAC enforcer applying - token := r.Header.Get("token") - if ok := impl.enforcer.Enforce(token, casbin.ResourceMigrate, casbin.ActionCreate, bean.Name); !ok { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) - return - } - //RBAC enforcer Ends - - res, err := impl.dbConfigService.Save(&bean) - if err != nil { - impl.logger.Errorw("service err, SaveDbConfig", "err", err, "payload", bean) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - common.WriteJsonResp(w, err, res, http.StatusOK) -} - -func (impl MigrateDbRestHandlerImpl) FetchAllDbConfig(w http.ResponseWriter, r *http.Request) { - res, err := impl.dbConfigService.GetAll() - if err != nil { - impl.logger.Errorw("service err, FetchAllDbConfig", "err", err) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - - // RBAC enforcer applying - token := r.Header.Get("token") - var result []types.DbConfigBean - for _, item := range res { - if ok := impl.enforcer.Enforce(token, casbin.ResourceMigrate, casbin.ActionGet, item.Name); ok { - result = append(result, *item) - } - } - //RBAC enforcer Ends - - common.WriteJsonResp(w, err, result, http.StatusOK) -} - -func (impl MigrateDbRestHandlerImpl) FetchOneDbConfig(w http.ResponseWriter, r *http.Request) { - params := mux.Vars(r) - id, err := strconv.Atoi(params["id"]) - if err != nil { - impl.logger.Errorw("request err, FetchOneDbConfig", "err", err, "id", id) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - res, err := impl.dbConfigService.GetById(id) - if err != nil { - impl.logger.Errorw("service err, FetchOneDbConfig", "err", err, "id", id) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - - // RBAC enforcer applying - token := r.Header.Get("token") - if ok := impl.enforcer.Enforce(token, casbin.ResourceMigrate, casbin.ActionGet, res.Name); !ok { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) - return - } - //RBAC enforcer Ends - - common.WriteJsonResp(w, err, res, http.StatusOK) -} - -func (impl MigrateDbRestHandlerImpl) UpdateDbConfig(w http.ResponseWriter, r *http.Request) { - decoder := json.NewDecoder(r.Body) - userId, err := impl.userAuthService.GetLoggedInUser(r) - if userId == 0 || err != nil { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return - } - var bean types.DbConfigBean - - err = decoder.Decode(&bean) - if err != nil { - impl.logger.Errorw("request err, UpdateDbConfig", "err", err, "payload", bean) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - bean.UserId = userId - impl.logger.Errorw("request payload, UpdateDbConfig", "err", err, "payload", bean) - err = impl.validator.Struct(bean) - if err != nil { - impl.logger.Errorw("service err, UpdateDbConfig", "err", err, "payload", bean) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - - // RBAC enforcer applying - token := r.Header.Get("token") - if ok := impl.enforcer.Enforce(token, casbin.ResourceMigrate, casbin.ActionUpdate, bean.Name); !ok { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) - return - } - //RBAC enforcer Ends - - res, err := impl.dbConfigService.Update(&bean) - if err != nil { - impl.logger.Errorw("service err, UpdateDbConfig", "err", err, "payload", bean) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - common.WriteJsonResp(w, err, res, http.StatusOK) -} - -func (impl MigrateDbRestHandlerImpl) FetchDbConfigForAutoComp(w http.ResponseWriter, r *http.Request) { - res, err := impl.dbConfigService.GetForAutocomplete() - if err != nil { - impl.logger.Errorw("service err, FetchDbConfigForAutoComp", "err", err) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - - // RBAC enforcer applying - token := r.Header.Get("token") - var result []types.DbConfigBean - for _, item := range res { - if ok := impl.enforcer.Enforce(token, casbin.ResourceMigrate, casbin.ActionGet, item.Name); ok { - result = append(result, *item) - } - } - //RBAC enforcer Ends - - common.WriteJsonResp(w, err, result, http.StatusOK) -} diff --git a/api/restHandler/NotificationRestHandler.go b/api/restHandler/NotificationRestHandler.go index 9840f69ec4..996ecd52e9 100644 --- a/api/restHandler/NotificationRestHandler.go +++ b/api/restHandler/NotificationRestHandler.go @@ -73,9 +73,8 @@ type NotificationRestHandlerImpl struct { dockerRegistryConfig pipeline.DockerRegistryConfig logger *zap.SugaredLogger gitRegistryConfig pipeline.GitRegistryConfig - dbConfigService pipeline.DbConfigService - userAuthService user.UserService - validator *validator.Validate + userAuthService user.UserService + validator *validator.Validate notificationService notifier.NotificationConfigService slackService notifier.SlackNotificationService webhookService notifier.WebhookNotificationService @@ -94,7 +93,7 @@ type ChannelDto struct { func NewNotificationRestHandlerImpl(dockerRegistryConfig pipeline.DockerRegistryConfig, logger *zap.SugaredLogger, gitRegistryConfig pipeline.GitRegistryConfig, - dbConfigService pipeline.DbConfigService, userAuthService user.UserService, + userAuthService user.UserService, validator *validator.Validate, notificationService notifier.NotificationConfigService, slackService notifier.SlackNotificationService, webhookService notifier.WebhookNotificationService, sesService notifier.SESNotificationService, smtpService notifier.SMTPNotificationService, enforcer casbin.Enforcer, teamService team.TeamService, environmentService cluster.EnvironmentService, pipelineBuilder pipeline.PipelineBuilder, @@ -103,7 +102,6 @@ func NewNotificationRestHandlerImpl(dockerRegistryConfig pipeline.DockerRegistry dockerRegistryConfig: dockerRegistryConfig, logger: logger, gitRegistryConfig: gitRegistryConfig, - dbConfigService: dbConfigService, userAuthService: userAuthService, validator: validator, notificationService: notificationService, diff --git a/api/restHandler/PolicyRestHandler.go b/api/restHandler/PolicyRestHandler.go index c3f99131d9..81e133c79f 100644 --- a/api/restHandler/PolicyRestHandler.go +++ b/api/restHandler/PolicyRestHandler.go @@ -43,10 +43,10 @@ type PolicyRestHandler interface { } type PolicyRestHandlerImpl struct { logger *zap.SugaredLogger - policyService security.PolicyService - userService user2.UserService - userAuthService user2.UserAuthService - enforcer casbin.Enforcer + policyService security.PolicyService + userService user2.UserService + userAuthService user2.UserAuthService + enforcer casbin.Enforcer enforcerUtil rbac.EnforcerUtil environmentService cluster.EnvironmentService } diff --git a/api/restHandler/TelemetryRestHandler.go b/api/restHandler/TelemetryRestHandler.go index eb1cd76821..d6406b967b 100644 --- a/api/restHandler/TelemetryRestHandler.go +++ b/api/restHandler/TelemetryRestHandler.go @@ -38,8 +38,8 @@ type TelemetryRestHandler interface { type TelemetryRestHandlerImpl struct { logger *zap.SugaredLogger telemetryEventClient telemetry.TelemetryEventClient - enforcer casbin.Enforcer - userService user.UserService + enforcer casbin.Enforcer + userService user.UserService } type TelemetryGenericEvent struct { diff --git a/api/restHandler/TestSuitRestHandler.go b/api/restHandler/TestSuitRestHandler.go deleted file mode 100644 index 968a3f380d..0000000000 --- a/api/restHandler/TestSuitRestHandler.go +++ /dev/null @@ -1,276 +0,0 @@ -/* - * Copyright (c) 2020 Devtron Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package restHandler - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "strconv" - - "github.com/devtron-labs/devtron/api/restHandler/common" - "github.com/devtron-labs/devtron/client/events" - "github.com/devtron-labs/devtron/client/grafana" - "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" - "github.com/devtron-labs/devtron/pkg/auth/user" - "github.com/devtron-labs/devtron/util/rbac" - "github.com/gorilla/mux" - "go.uber.org/zap" - "gopkg.in/go-playground/validator.v9" -) - -type TestSuitRestHandler interface { - SuitesProxy(w http.ResponseWriter, r *http.Request) - GetTestSuites(w http.ResponseWriter, r *http.Request) - DetailedTestSuites(w http.ResponseWriter, r *http.Request) - GetAllSuitByID(w http.ResponseWriter, r *http.Request) - GetAllTestCases(w http.ResponseWriter, r *http.Request) - GetTestCaseByID(w http.ResponseWriter, r *http.Request) - RedirectTriggerForApp(w http.ResponseWriter, r *http.Request) - RedirectTriggerForEnv(w http.ResponseWriter, r *http.Request) -} - -type TestSuitRestHandlerImpl struct { - logger *zap.SugaredLogger - userService user.UserService - validator *validator.Validate - enforcer casbin.Enforcer - enforcerUtil rbac.EnforcerUtil - config *client.EventClientConfig - client *http.Client -} - -func NewTestSuitRestHandlerImpl(logger *zap.SugaredLogger, userService user.UserService, - validator *validator.Validate, enforcer casbin.Enforcer, enforcerUtil rbac.EnforcerUtil, - config *client.EventClientConfig, client *http.Client) *TestSuitRestHandlerImpl { - return &TestSuitRestHandlerImpl{ - logger: logger, - userService: userService, - enforcer: enforcer, - enforcerUtil: enforcerUtil, - config: config, - client: client, - } -} - -type TestSuiteBean struct { - Link string `json:"link,omitempty"` - PipelineId int `json:"PipelineId"` - TriggerId int `json:"triggerId"` -} - -func (impl TestSuitRestHandlerImpl) SuitesProxy(w http.ResponseWriter, r *http.Request) { - userId, err := impl.userService.GetLoggedInUser(r) - impl.logger.Debugw("request for user", "userId", userId) - if userId == 0 || err != nil { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return - } - decoder := json.NewDecoder(r.Body) - var bean TestSuiteBean - err = decoder.Decode(&bean) - if err != nil { - impl.logger.Errorw("decode err", "err", err) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - - link := fmt.Sprintf("%s/%s", impl.config.TestSuitURL, bean.Link) - res, err := impl.HttpGet(link) - if err != nil { - impl.logger.Error(err) - } - common.WriteJsonResp(w, err, res, http.StatusOK) -} - -func (impl TestSuitRestHandlerImpl) GetTestSuites(w http.ResponseWriter, r *http.Request) { - userId, err := impl.userService.GetLoggedInUser(r) - impl.logger.Debugw("request for user", "userId", userId) - if userId == 0 || err != nil { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return - } - - link := fmt.Sprintf("%s/%s", impl.config.TestSuitURL, "testsuite") - res, err := impl.HttpGet(link) - if err != nil { - impl.logger.Error(err) - } - common.WriteJsonResp(w, err, res, http.StatusOK) -} - -func (impl TestSuitRestHandlerImpl) DetailedTestSuites(w http.ResponseWriter, r *http.Request) { - userId, err := impl.userService.GetLoggedInUser(r) - impl.logger.Debugw("request for user", "userId", userId) - if userId == 0 || err != nil { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return - } - - link := fmt.Sprintf("%s/%s", impl.config.TestSuitURL, "testsuites/all") - res, err := impl.HttpGet(link) - if err != nil { - impl.logger.Error(err) - } - common.WriteJsonResp(w, err, res, http.StatusOK) -} - -func (impl TestSuitRestHandlerImpl) GetAllSuitByID(w http.ResponseWriter, r *http.Request) { - userId, err := impl.userService.GetLoggedInUser(r) - impl.logger.Debugw("request for user", "userId", userId) - if userId == 0 || err != nil { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return - } - vars := mux.Vars(r) - id, err := strconv.Atoi(vars["pipelineId"]) - if err != nil { - impl.logger.Error(err) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - link := fmt.Sprintf("%s/%s/%d", impl.config.TestSuitURL, "testsuite", id) - res, err := impl.HttpGet(link) - if err != nil { - impl.logger.Error(err) - } - common.WriteJsonResp(w, err, res, http.StatusOK) -} - -func (impl TestSuitRestHandlerImpl) GetAllTestCases(w http.ResponseWriter, r *http.Request) { - userId, err := impl.userService.GetLoggedInUser(r) - impl.logger.Debugw("request for user", "userId", userId) - if userId == 0 || err != nil { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return - } - - link := fmt.Sprintf("%s/%s", impl.config.TestSuitURL, "testcase") - res, err := impl.HttpGet(link) - if err != nil { - impl.logger.Error(err) - } - common.WriteJsonResp(w, err, res, http.StatusOK) -} - -func (impl TestSuitRestHandlerImpl) GetTestCaseByID(w http.ResponseWriter, r *http.Request) { - userId, err := impl.userService.GetLoggedInUser(r) - impl.logger.Debugw("request for user", "userId", userId) - if userId == 0 || err != nil { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return - } - vars := mux.Vars(r) - id, err := strconv.Atoi(vars["pipelineId"]) - if err != nil { - impl.logger.Error(err) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - link := fmt.Sprintf("%s/%s/%d", impl.config.TestSuitURL, "testcase", id) - res, err := impl.HttpGet(link) - if err != nil { - impl.logger.Error(err) - } - common.WriteJsonResp(w, err, res, http.StatusOK) -} - -func (impl TestSuitRestHandlerImpl) RedirectTriggerForApp(w http.ResponseWriter, r *http.Request) { - userId, err := impl.userService.GetLoggedInUser(r) - impl.logger.Debugw("request for user", "userId", userId) - if userId == 0 || err != nil { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return - } - vars := mux.Vars(r) - appId, err := strconv.Atoi(vars["pipelineId"]) - if err != nil { - impl.logger.Error(err) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - link := fmt.Sprintf("%s/%s/%d", impl.config.TestSuitURL, "triggers", appId) - impl.logger.Debugw("redirect to link", "link", link) - res, err := impl.HttpGet(link) - if err != nil { - impl.logger.Error(err) - } - common.WriteJsonResp(w, err, res, http.StatusOK) -} - -func (impl TestSuitRestHandlerImpl) RedirectTriggerForEnv(w http.ResponseWriter, r *http.Request) { - userId, err := impl.userService.GetLoggedInUser(r) - impl.logger.Debugw("request for user", "userId", userId) - if userId == 0 || err != nil { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return - } - - vars := mux.Vars(r) - appId, err := strconv.Atoi(vars["pipelineId"]) - if err != nil { - impl.logger.Error(err) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - envId, err := strconv.Atoi(vars["triggerId"]) - if err != nil { - impl.logger.Error(err) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - - link := fmt.Sprintf("%s/%s/%d/%d", impl.config.TestSuitURL, "triggers", appId, envId) - res, err := impl.HttpGet(link) - if err != nil { - impl.logger.Error(err) - } - common.WriteJsonResp(w, err, res, http.StatusOK) -} - -func (impl TestSuitRestHandlerImpl) HttpGet(url string) (map[string]interface{}, error) { - req, err := http.NewRequest(http.MethodGet, url, nil) - if err != nil { - impl.logger.Errorw("error while fetching data", "err", err) - return nil, err - } - resp, err := impl.client.Do(req) - if err != nil { - impl.logger.Errorw("err", err) - return nil, err - } - status := grafana.StatusCode(resp.StatusCode) - resBody, err := ioutil.ReadAll(resp.Body) - var apiRes map[string]interface{} - if status.IsSuccess() { - if err != nil { - impl.logger.Errorw("error in grafana communication ", "err", err) - return nil, err - } - err = json.Unmarshal(resBody, &apiRes) - if err != nil { - impl.logger.Errorw("error in grafana resp unmarshalling ", "err", err) - return nil, err - } - } else { - impl.logger.Errorw("api err", "res", string(resBody)) - return nil, fmt.Errorf("res not success, code: %d ,response body: %s", status, string(resBody)) - } - return apiRes, nil -} diff --git a/api/restHandler/app/DeploymentPipelineRestHandler.go b/api/restHandler/app/DeploymentPipelineRestHandler.go index 8eedcc34b5..e7dfcac719 100644 --- a/api/restHandler/app/DeploymentPipelineRestHandler.go +++ b/api/restHandler/app/DeploymentPipelineRestHandler.go @@ -23,7 +23,6 @@ import ( "github.com/devtron-labs/devtron/pkg/generateManifest" "github.com/devtron-labs/devtron/pkg/pipeline" bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" - "github.com/devtron-labs/devtron/pkg/pipeline/types" resourceGroup2 "github.com/devtron-labs/devtron/pkg/resourceGroup" "github.com/devtron-labs/devtron/pkg/resourceQualifiers" "github.com/devtron-labs/devtron/pkg/variables/models" @@ -87,9 +86,6 @@ type DevtronAppDeploymentConfigRestHandler interface { } type DevtronAppPrePostDeploymentRestHandler interface { - GetMigrationConfig(w http.ResponseWriter, r *http.Request) - CreateMigrationConfig(w http.ResponseWriter, r *http.Request) - UpdateMigrationConfig(w http.ResponseWriter, r *http.Request) GetStageStatus(w http.ResponseWriter, r *http.Request) GetPrePostDeploymentLogs(w http.ResponseWriter, r *http.Request) // CancelStage Cancel Pre/Post ArgoWorkflow execution @@ -1523,133 +1519,6 @@ func (handler PipelineConfigRestHandlerImpl) GetArtifactsForRollback(w http.Resp common.WriteJsonResp(w, err, ciArtifactResponse, http.StatusOK) } -func (handler PipelineConfigRestHandlerImpl) GetMigrationConfig(w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) - pipelineId, err := strconv.Atoi(vars["pipelineId"]) - if err != nil { - handler.Logger.Errorw("request err, GetMigrationConfig", "err", err, "pipelineId", pipelineId) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - handler.Logger.Infow("request payload, GetMigrationConfig", "pipelineId", pipelineId) - token := r.Header.Get("token") - deploymentPipeline, err := handler.pipelineBuilder.FindPipelineById(pipelineId) - if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - app, err := handler.pipelineBuilder.GetApp(deploymentPipeline.AppId) - if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - resourceName := handler.enforcerUtil.GetAppRBACName(app.AppName) - if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionGet, resourceName); !ok { - common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden) - return - } - ciConf, err := handler.dbMigrationService.GetByPipelineId(pipelineId) - if err != nil { - handler.Logger.Errorw("service err, GetMigrationConfig", "err", err, "pipelineId", pipelineId) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - common.WriteJsonResp(w, err, ciConf, http.StatusOK) -} - -func (handler PipelineConfigRestHandlerImpl) CreateMigrationConfig(w http.ResponseWriter, r *http.Request) { - decoder := json.NewDecoder(r.Body) - userId, err := handler.userAuthService.GetLoggedInUser(r) - if userId == 0 || err != nil { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return - } - var dbMigrationConfigBean types.DbMigrationConfigBean - err = decoder.Decode(&dbMigrationConfigBean) - - dbMigrationConfigBean.UserId = userId - if err != nil { - handler.Logger.Errorw("request err, CreateMigrationConfig", "err", err, "payload", dbMigrationConfigBean) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - err = handler.validator.Struct(dbMigrationConfigBean) - if err != nil { - handler.Logger.Errorw("validation err, CreateMigrationConfig", "err", err, "payload", dbMigrationConfigBean) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - handler.Logger.Infow("request payload, CreateMigrationConfig", "payload", dbMigrationConfigBean) - token := r.Header.Get("token") - deploymentPipeline, err := handler.pipelineBuilder.FindPipelineById(dbMigrationConfigBean.PipelineId) - if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - app, err := handler.pipelineBuilder.GetApp(deploymentPipeline.AppId) - if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - } - resourceName := handler.enforcerUtil.GetAppRBACName(app.AppName) - if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, resourceName); !ok { - common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden) - return - } - createResp, err := handler.dbMigrationService.Save(&dbMigrationConfigBean) - if err != nil { - handler.Logger.Errorw("service err, CreateMigrationConfig", "err", err, "payload", dbMigrationConfigBean) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - common.WriteJsonResp(w, err, createResp, http.StatusOK) -} -func (handler PipelineConfigRestHandlerImpl) UpdateMigrationConfig(w http.ResponseWriter, r *http.Request) { - decoder := json.NewDecoder(r.Body) - userId, err := handler.userAuthService.GetLoggedInUser(r) - if userId == 0 || err != nil { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return - } - var dbMigrationConfigBean types.DbMigrationConfigBean - err = decoder.Decode(&dbMigrationConfigBean) - dbMigrationConfigBean.UserId = userId - if err != nil { - handler.Logger.Errorw("request err, UpdateMigrationConfig", "err", err, "payload", dbMigrationConfigBean) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - err = handler.validator.Struct(dbMigrationConfigBean) - if err != nil { - handler.Logger.Errorw("validation err, UpdateMigrationConfig", "err", err, "payload", dbMigrationConfigBean) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - handler.Logger.Infow("request payload, UpdateMigrationConfig", "payload", dbMigrationConfigBean) - token := r.Header.Get("token") - deploymentPipeline, err := handler.pipelineBuilder.FindPipelineById(dbMigrationConfigBean.PipelineId) - if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - app, err := handler.pipelineBuilder.GetApp(deploymentPipeline.AppId) - if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - resourceName := handler.enforcerUtil.GetAppRBACName(app.AppName) - if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, resourceName); !ok { - common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden) - return - } - createResp, err := handler.dbMigrationService.Update(&dbMigrationConfigBean) - if err != nil { - handler.Logger.Errorw("service err, UpdateMigrationConfig", "err", err, "payload", dbMigrationConfigBean) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - common.WriteJsonResp(w, err, createResp, http.StatusOK) -} - func (handler PipelineConfigRestHandlerImpl) EnvConfigOverrideReset(w http.ResponseWriter, r *http.Request) { userId, err := handler.userAuthService.GetLoggedInUser(r) if userId == 0 || err != nil { diff --git a/api/restHandler/app/PipelineConfigRestHandler.go b/api/restHandler/app/PipelineConfigRestHandler.go index cf88641e43..71eeeade08 100644 --- a/api/restHandler/app/PipelineConfigRestHandler.go +++ b/api/restHandler/app/PipelineConfigRestHandler.go @@ -108,7 +108,6 @@ type PipelineConfigRestHandlerImpl struct { Logger *zap.SugaredLogger chartService chart.ChartService propertiesConfigService pipeline.PropertiesConfigService - dbMigrationService pipeline.DbMigrationService application application.ServiceClient userAuthService user.UserService validator *validator.Validate @@ -137,7 +136,6 @@ type PipelineConfigRestHandlerImpl struct { func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger *zap.SugaredLogger, chartService chart.ChartService, propertiesConfigService pipeline.PropertiesConfigService, - dbMigrationService pipeline.DbMigrationService, application application.ServiceClient, userAuthService user.UserService, teamService team.TeamService, @@ -167,7 +165,6 @@ func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger Logger: Logger, chartService: chartService, propertiesConfigService: propertiesConfigService, - dbMigrationService: dbMigrationService, application: application, userAuthService: userAuthService, validator: validator, diff --git a/api/router/MigrateDbRouter.go b/api/router/MigrateDbRouter.go deleted file mode 100644 index 1652ca694a..0000000000 --- a/api/router/MigrateDbRouter.go +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2020 Devtron Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package router - -import ( - "github.com/devtron-labs/devtron/api/restHandler" - "github.com/gorilla/mux" -) - -type MigrateDbRouter interface { - InitMigrateDbRouter(migrateRouter *mux.Router) -} -type MigrateDbRouterImpl struct { - migrateDbRestHandler restHandler.MigrateDbRestHandler -} - -func NewMigrateDbRouterImpl(migrateDbRestHandler restHandler.MigrateDbRestHandler) *MigrateDbRouterImpl { - return &MigrateDbRouterImpl{migrateDbRestHandler: migrateDbRestHandler} -} -func (impl MigrateDbRouterImpl) InitMigrateDbRouter(migrateRouter *mux.Router) { - migrateRouter.Path("/db"). - HandlerFunc(impl.migrateDbRestHandler.SaveDbConfig). - Methods("POST") - migrateRouter.Path("/db"). - HandlerFunc(impl.migrateDbRestHandler.FetchAllDbConfig). - Methods("GET") - migrateRouter.Path("/db/{id}"). - HandlerFunc(impl.migrateDbRestHandler.FetchOneDbConfig). - Methods("GET") - migrateRouter.Path("/db"). - HandlerFunc(impl.migrateDbRestHandler.UpdateDbConfig). - Methods("PUT") - migrateRouter.Path("/db/autocomplete"). - HandlerFunc(impl.migrateDbRestHandler.FetchDbConfigForAutoComp). - Methods("GET") -} diff --git a/api/router/PipelineConfigRouter.go b/api/router/PipelineConfigRouter.go index 54bc7613ec..1f1339645b 100644 --- a/api/router/PipelineConfigRouter.go +++ b/api/router/PipelineConfigRouter.go @@ -94,10 +94,6 @@ func (router PipelineConfigRouterImpl) initPipelineConfigRouter(configRouter *mu configRouter.Path("/cd-pipeline/{cd_pipeline_id}/material").HandlerFunc(router.restHandler.GetArtifactsByCDPipeline).Methods("GET") configRouter.Path("/cd-pipeline/{cd_pipeline_id}/material/rollback").HandlerFunc(router.restHandler.GetArtifactsForRollback).Methods("GET") - configRouter.Path("/migrate/db").HandlerFunc(router.restHandler.CreateMigrationConfig).Methods("POST") - configRouter.Path("/migrate/db/update").HandlerFunc(router.restHandler.UpdateMigrationConfig).Methods("POST") - configRouter.Path("/migrate/db/{pipelineId}").HandlerFunc(router.restHandler.GetMigrationConfig).Methods("GET") - configRouter.Path("/team/by-id/{teamId}").HandlerFunc(router.restHandler.FindAppsByTeamId).Methods("GET") configRouter.Path("/team/by-name/{teamName}").HandlerFunc(router.restHandler.FindAppsByTeamName).Methods("GET") diff --git a/api/router/ProjectManagementRouter.go b/api/router/ProjectManagementRouter.go deleted file mode 100644 index 35247a1d2a..0000000000 --- a/api/router/ProjectManagementRouter.go +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2020 Devtron Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package router - -import ( - "github.com/devtron-labs/devtron/api/restHandler" - "github.com/gorilla/mux" -) - -type ProjectManagementRouter interface { - InitProjectManagementRouter(jiraRouter *mux.Router) -} - -type ProjectManagementRouterImpl struct { - jiraRestHandler restHandler.JiraRestHandler -} - -func NewProjectManagementRouterImpl(jiraRestHandler restHandler.JiraRestHandler) *ProjectManagementRouterImpl { - return &ProjectManagementRouterImpl{jiraRestHandler: jiraRestHandler} -} - -func (impl ProjectManagementRouterImpl) InitProjectManagementRouter(jiraRouter *mux.Router) { - jiraRouter.Path("/jira/config"). - Methods("POST"). - HandlerFunc(impl.jiraRestHandler.SaveAccountConfig) - jiraRouter.Path("/jira/release"). - Methods("POST"). - HandlerFunc(impl.jiraRestHandler.UpdateIssueStatus) - -} diff --git a/api/router/TestSuites.go b/api/router/TestSuites.go deleted file mode 100644 index a1ad48af12..0000000000 --- a/api/router/TestSuites.go +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2020 Devtron Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package router - -import ( - "github.com/devtron-labs/devtron/api/restHandler" - "github.com/gorilla/mux" -) - -type TestSuitRouter interface { - InitTestSuitRouter(gocdRouter *mux.Router) -} -type TestSuitRouterImpl struct { - testSuitRouter restHandler.TestSuitRestHandler -} - -func NewTestSuitRouterImpl(testSuitRouter restHandler.TestSuitRestHandler) *TestSuitRouterImpl { - return &TestSuitRouterImpl{testSuitRouter: testSuitRouter} -} - -func (impl TestSuitRouterImpl) InitTestSuitRouter(configRouter *mux.Router) { - configRouter.Path("/suites/proxy").HandlerFunc(impl.testSuitRouter.SuitesProxy).Methods("POST") - configRouter.Path("/suites/list").HandlerFunc(impl.testSuitRouter.GetTestSuites).Methods("GET") - configRouter.Path("/suites/list/detail").HandlerFunc(impl.testSuitRouter.DetailedTestSuites).Methods("GET") - configRouter.Path("/suites/{pipelineId}").HandlerFunc(impl.testSuitRouter.GetAllSuitByID).Methods("GET") - configRouter.Path("/cases").HandlerFunc(impl.testSuitRouter.GetAllTestCases).Methods("GET") - configRouter.Path("/cases/{pipelineId}").HandlerFunc(impl.testSuitRouter.GetTestCaseByID).Methods("GET") - configRouter.Path("/trigger/{pipelineId}").HandlerFunc(impl.testSuitRouter.RedirectTriggerForApp).Methods("GET") - configRouter.Path("/trigger/{pipelineId}/{triggerId}").HandlerFunc(impl.testSuitRouter.RedirectTriggerForEnv).Methods("GET") -} diff --git a/api/router/router.go b/api/router/router.go index 78b69fcfc8..558940e3f4 100644 --- a/api/router/router.go +++ b/api/router/router.go @@ -57,7 +57,6 @@ type MuxRouter struct { HelmRouter PipelineTriggerRouter PipelineConfigRouter PipelineConfigRouter JobRouter JobRouter - MigrateDbRouter MigrateDbRouter EnvironmentClusterMappingsRouter cluster.EnvironmentRouter AppListingRouter AppListingRouter ClusterRouter cluster.ClusterRouter @@ -65,7 +64,6 @@ type MuxRouter struct { UserAuthRouter user.UserAuthRouter ApplicationRouter ApplicationRouter CDRouter CDRouter - ProjectManagementRouter ProjectManagementRouter GitProviderRouter GitProviderRouter GitHostRouter GitHostRouter DockerRegRouter DockerRegRouter @@ -85,7 +83,6 @@ type MuxRouter struct { deploymentGroupRouter DeploymentGroupRouter chartGroupRouter ChartGroupRouter batchOperationRouter BatchOperationRouter - testSuitRouter TestSuitRouter imageScanRouter ImageScanRouter policyRouter PolicyRouter gitOpsConfigRouter GitOpsConfigRouter @@ -125,10 +122,10 @@ type MuxRouter struct { } func NewMuxRouter(logger *zap.SugaredLogger, HelmRouter PipelineTriggerRouter, PipelineConfigRouter PipelineConfigRouter, - MigrateDbRouter MigrateDbRouter, AppListingRouter AppListingRouter, + AppListingRouter AppListingRouter, EnvironmentClusterMappingsRouter cluster.EnvironmentRouter, ClusterRouter cluster.ClusterRouter, WebHookRouter WebhookRouter, UserAuthRouter user.UserAuthRouter, ApplicationRouter ApplicationRouter, - CDRouter CDRouter, ProjectManagementRouter ProjectManagementRouter, + CDRouter CDRouter, GitProviderRouter GitProviderRouter, GitHostRouter GitHostRouter, DockerRegRouter DockerRegRouter, NotificationRouter NotificationRouter, @@ -139,7 +136,7 @@ func NewMuxRouter(logger *zap.SugaredLogger, HelmRouter PipelineTriggerRouter, P ciEventHandler pubsub.CiEventHandler, pubsubClient *pubsub2.PubSubClientServiceImpl, UserRouter user.UserRouter, ChartRefRouter ChartRefRouter, ConfigMapRouter ConfigMapRouter, AppStoreRouter appStore.AppStoreRouter, chartRepositoryRouter chartRepo.ChartRepositoryRouter, ReleaseMetricsRouter ReleaseMetricsRouter, deploymentGroupRouter DeploymentGroupRouter, batchOperationRouter BatchOperationRouter, - chartGroupRouter ChartGroupRouter, testSuitRouter TestSuitRouter, imageScanRouter ImageScanRouter, + chartGroupRouter ChartGroupRouter, imageScanRouter ImageScanRouter, policyRouter PolicyRouter, gitOpsConfigRouter GitOpsConfigRouter, dashboardRouter dashboard.DashboardRouter, attributesRouter AttributesRouter, userAttributesRouter UserAttributesRouter, commonRouter CommonRouter, grafanaRouter GrafanaRouter, ssoLoginRouter sso.SsoLoginRouter, telemetryRouter TelemetryRouter, telemetryWatcher telemetry.TelemetryEventClient, bulkUpdateRouter BulkUpdateRouter, webhookListenerRouter WebhookListenerRouter, appRouter AppRouter, coreAppRouter CoreAppRouter, helmAppRouter client.HelmAppRouter, k8sApplicationRouter application.K8sApplicationRouter, @@ -158,7 +155,6 @@ func NewMuxRouter(logger *zap.SugaredLogger, HelmRouter PipelineTriggerRouter, P Router: mux.NewRouter(), HelmRouter: HelmRouter, PipelineConfigRouter: PipelineConfigRouter, - MigrateDbRouter: MigrateDbRouter, EnvironmentClusterMappingsRouter: EnvironmentClusterMappingsRouter, AppListingRouter: AppListingRouter, ClusterRouter: ClusterRouter, @@ -166,7 +162,6 @@ func NewMuxRouter(logger *zap.SugaredLogger, HelmRouter PipelineTriggerRouter, P UserAuthRouter: UserAuthRouter, ApplicationRouter: ApplicationRouter, CDRouter: CDRouter, - ProjectManagementRouter: ProjectManagementRouter, DockerRegRouter: DockerRegRouter, GitProviderRouter: GitProviderRouter, GitHostRouter: GitHostRouter, @@ -187,7 +182,6 @@ func NewMuxRouter(logger *zap.SugaredLogger, HelmRouter PipelineTriggerRouter, P deploymentGroupRouter: deploymentGroupRouter, batchOperationRouter: batchOperationRouter, chartGroupRouter: chartGroupRouter, - testSuitRouter: testSuitRouter, imageScanRouter: imageScanRouter, policyRouter: policyRouter, gitOpsConfigRouter: gitOpsConfigRouter, @@ -275,9 +269,6 @@ func (r MuxRouter) Init() { jobConfigRouter := r.Router.PathPrefix("/orchestrator/job").Subrouter() r.JobRouter.InitJobRouter(jobConfigRouter) - migrateRouter := r.Router.PathPrefix("/orchestrator/migrate").Subrouter() - r.MigrateDbRouter.InitMigrateDbRouter(migrateRouter) - environmentClusterMappingsRouter := r.Router.PathPrefix("/orchestrator/env").Subrouter() r.EnvironmentClusterMappingsRouter.InitEnvironmentClusterMappingsRouter(environmentClusterMappingsRouter) r.resourceGroupingRouter.InitResourceGroupingRouter(environmentClusterMappingsRouter) @@ -294,9 +285,6 @@ func (r MuxRouter) Init() { rootRouter := r.Router.PathPrefix("/orchestrator").Subrouter() r.UserAuthRouter.InitUserAuthRouter(rootRouter) - projectManagementRouter := r.Router.PathPrefix("/orchestrator/project-management").Subrouter() - r.ProjectManagementRouter.InitProjectManagementRouter(projectManagementRouter) - gitRouter := r.Router.PathPrefix("/orchestrator/git").Subrouter() r.GitProviderRouter.InitGitProviderRouter(gitRouter) r.GitHostRouter.InitGitHostRouter(gitRouter) @@ -336,9 +324,6 @@ func (r MuxRouter) Init() { chartGroupRouter := r.Router.PathPrefix("/orchestrator/chart-group").Subrouter() r.chartGroupRouter.initChartGroupRouter(chartGroupRouter) - testSuitRouter := r.Router.PathPrefix("/orchestrator/test-report").Subrouter() - r.testSuitRouter.InitTestSuitRouter(testSuitRouter) - imageScanRouter := r.Router.PathPrefix("/orchestrator/security/scan").Subrouter() r.imageScanRouter.InitImageScanRouter(imageScanRouter) diff --git a/api/terminal/UserTerminalAccessRestHandler.go b/api/terminal/UserTerminalAccessRestHandler.go index c2cb455c9e..4b71273c97 100644 --- a/api/terminal/UserTerminalAccessRestHandler.go +++ b/api/terminal/UserTerminalAccessRestHandler.go @@ -39,9 +39,9 @@ type validShellResponse struct { type UserTerminalAccessRestHandlerImpl struct { Logger *zap.SugaredLogger UserTerminalAccessService clusterTerminalAccess.UserTerminalAccessService - Enforcer casbin.Enforcer - UserService user.UserService - validator *validator.Validate + Enforcer casbin.Enforcer + UserService user.UserService + validator *validator.Validate } func NewUserTerminalAccessRestHandlerImpl(logger *zap.SugaredLogger, userTerminalAccessService clusterTerminalAccess.UserTerminalAccessService, Enforcer casbin.Enforcer, diff --git a/client/events/EventBuilder.go b/client/events/EventBuilder.go index d3c8150017..63060238ea 100644 --- a/client/events/EventBuilder.go +++ b/client/events/EventBuilder.go @@ -49,9 +49,9 @@ type EventSimpleFactoryImpl struct { ciWorkflowRepository pipelineConfig.CiWorkflowRepository ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository ciPipelineRepository pipelineConfig.CiPipelineRepository - pipelineRepository pipelineConfig.PipelineRepository - userRepository repository.UserRepository - ciArtifactRepository repository2.CiArtifactRepository + pipelineRepository pipelineConfig.PipelineRepository + userRepository repository.UserRepository + ciArtifactRepository repository2.CiArtifactRepository } func NewEventSimpleFactoryImpl(logger *zap.SugaredLogger, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, diff --git a/client/events/EventClient.go b/client/events/EventClient.go index 9fa5e73591..b32a06c9ff 100644 --- a/client/events/EventClient.go +++ b/client/events/EventClient.go @@ -39,7 +39,6 @@ import ( type EventClientConfig struct { DestinationURL string `env:"EVENT_URL" envDefault:"http://localhost:3000/notify"` - TestSuitURL string `env:"TEST_SUIT_URL" envDefault:"http://localhost:3000"` } func GetEventClientConfig() (*EventClientConfig, error) { @@ -54,7 +53,6 @@ func GetEventClientConfig() (*EventClientConfig, error) { type EventClient interface { WriteNotificationEvent(event Event) (bool, error) WriteNatsEvent(channel string, payload interface{}) error - SendTestSuite(reqBody []byte) (bool, error) } type Event struct { @@ -274,20 +272,3 @@ func (impl *EventRESTClientImpl) WriteNatsEvent(topic string, payload interface{ err = impl.pubsubClient.Publish(topic, string(body)) return err } - -func (impl *EventRESTClientImpl) SendTestSuite(reqBody []byte) (bool, error) { - impl.logger.Debugw("request", "body", string(reqBody)) - req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/triggers", impl.config.TestSuitURL), bytes.NewBuffer(reqBody)) - if err != nil { - impl.logger.Errorw("error while writing test suites", "err", err) - return false, err - } - req.Header.Set("Content-Type", "application/json") - resp, err := impl.client.Do(req) - if err != nil { - impl.logger.Errorw("error while UpdateJiraTransition request ", "err", err) - return false, err - } - impl.logger.Debugw("response from test suit create api", "status code", resp.StatusCode) - return true, err -} diff --git a/internal/sql/repository/DbConfigRepository.go b/internal/sql/repository/DbConfigRepository.go deleted file mode 100644 index e5a26249c8..0000000000 --- a/internal/sql/repository/DbConfigRepository.go +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright (c) 2020 Devtron Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package repository - -import ( - "github.com/devtron-labs/devtron/pkg/sql" - "github.com/go-pg/pg" - "go.uber.org/zap" -) - -type DbType string - -const ( - Db_TYPE_POSTGRESS DbType = "postgres" - Db_TYPE_MYSQL DbType = "mysql" - DB_TYPE_MARIADB DbType = "mariadb" -) - -func (t DbType) IsValid() bool { - types := map[string]DbType{"postgres": Db_TYPE_POSTGRESS, "mysql": Db_TYPE_MYSQL, "mariadb": DB_TYPE_MARIADB} - _, ok := types[string(t)] - return ok -} - -type DbConfig struct { - tableName struct{} `sql:"db_config" pg:",discard_unknown_columns"` - Id int `sql:"id,pk"` - Name string `sql:"name"` //name by which user identifies this db - Type DbType `sql:"type"` //type of db, PG, MYsql, MariaDb - Host string `sql:"host"` - Port string `sql:"port"` - DbName string `sql:"db_name"` //name of database inside PG - UserName string `sql:"user_name"` - Password string `sql:"password"` - Active bool `sql:"active"` - sql.AuditLog -} - -type DbConfigRepository interface { - Save(config *DbConfig) error - GetAll() (configs []*DbConfig, err error) - GetById(id int) (*DbConfig, error) - Update(config *DbConfig) (*DbConfig, error) - GetActiveForAutocomplete() (configs []*DbConfig, err error) -} -type DbConfigRepositoryImpl struct { - dbConnection *pg.DB - logger *zap.SugaredLogger -} - -func NewDbConfigRepositoryImpl(dbConnection *pg.DB, logger *zap.SugaredLogger) *DbConfigRepositoryImpl { - return &DbConfigRepositoryImpl{ - dbConnection: dbConnection, - logger: logger, - } -} - -func (impl DbConfigRepositoryImpl) Save(config *DbConfig) error { - return impl.dbConnection.Insert(config) -} - -func (impl DbConfigRepositoryImpl) GetAll() (configs []*DbConfig, err error) { - err = impl.dbConnection.Model(&configs).Select() - return configs, err -} - -func (impl DbConfigRepositoryImpl) GetById(id int) (*DbConfig, error) { - cfg := &DbConfig{Id: id} - err := impl.dbConnection.Model(cfg).WherePK().Select() - return cfg, err -} - -func (impl DbConfigRepositoryImpl) Update(config *DbConfig) (*DbConfig, error) { - _, err := impl.dbConnection.Model(config).WherePK().UpdateNotNull() - return config, err -} - -func (impl DbConfigRepositoryImpl) GetActiveForAutocomplete() (configs []*DbConfig, err error) { - err = impl.dbConnection.Model(&configs). - Where("active = ?", true). - Column("id", "name"). - Select() - - return configs, err -} diff --git a/internal/sql/repository/JiraAccountRepository.go b/internal/sql/repository/JiraAccountRepository.go deleted file mode 100644 index a831db6c8e..0000000000 --- a/internal/sql/repository/JiraAccountRepository.go +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright (c) 2020 Devtron Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package repository - -import ( - "github.com/devtron-labs/devtron/pkg/sql" - "github.com/go-pg/pg" - "time" -) - -type JiraAccountDetails struct { - tableName struct{} `sql:"project_management_tool_config" pg:",discard_unknown_columns"` - Id int32 `sql:"id,pk"` - UserName string `sql:"user_name"` - AccountURL string `sql:"account_url"` - AuthToken string `sql:"auth_token"` - CommitMessageRegex string `sql:"commit_message_regex"` - FinalIssueStatus string `sql:"final_issue_status"` - PipelineStage string `sql:"pipeline_stage"` - PipelineId int32 `sql:"pipeline_id"` - sql.AuditLog -} - -type JiraAccountRepository interface { - Save(accountDetails *JiraAccountDetails) error - FindByPipelineIdAndStage(pipelineId int32, pipelineStage string) (*JiraAccountDetails, error) - Update(accountDetails *JiraAccountDetails) error -} - -type JiraAccountRepositoryImpl struct { - dbConnection *pg.DB -} - -func NewJiraAccountRepositoryImpl(dbConnection *pg.DB) *JiraAccountRepositoryImpl { - return &JiraAccountRepositoryImpl{dbConnection: dbConnection} -} - -func (impl *JiraAccountRepositoryImpl) FindByPipelineIdAndStage(pipelineId int32, pipelineStage string) (*JiraAccountDetails, error) { - details := &JiraAccountDetails{} - err := impl.dbConnection.Model(details).Where("pipeline_id = ?", pipelineId).Where("pipeline_stage = ?", pipelineStage).Select() - return details, err -} - -func (impl *JiraAccountRepositoryImpl) Save(jiraAccountDetails *JiraAccountDetails) error { - model, err := impl.FindByPipelineIdAndStage(jiraAccountDetails.PipelineId, jiraAccountDetails.PipelineStage) - if err == nil && model != nil { - impl.buildAccountUpdateModel(jiraAccountDetails, model) - return impl.Update(model) - } - return impl.dbConnection.Insert(jiraAccountDetails) -} - -func (impl *JiraAccountRepositoryImpl) Update(jiraAccountDetails *JiraAccountDetails) error { - return impl.dbConnection.Update(jiraAccountDetails) -} - -func (impl *JiraAccountRepositoryImpl) buildAccountUpdateModel(jiraAccountDetails *JiraAccountDetails, model *JiraAccountDetails) { - if jiraAccountDetails.AccountURL != "" { - model.AccountURL = jiraAccountDetails.AccountURL - } - if jiraAccountDetails.AuthToken != "" { - model.AuthToken = jiraAccountDetails.AuthToken - } - if jiraAccountDetails.UserName != "" { - model.UserName = jiraAccountDetails.UserName - } - if jiraAccountDetails.CommitMessageRegex != "" { - model.CommitMessageRegex = jiraAccountDetails.CommitMessageRegex - } - if jiraAccountDetails.PipelineId != 0 { - model.PipelineId = jiraAccountDetails.PipelineId - } - if jiraAccountDetails.PipelineStage != "" { - model.PipelineStage = jiraAccountDetails.PipelineStage - } - model.UpdatedOn = time.Now() -} diff --git a/internal/sql/repository/pipelineConfig/DbMigrationConfig.go b/internal/sql/repository/pipelineConfig/DbMigrationConfig.go deleted file mode 100644 index 500a63d95f..0000000000 --- a/internal/sql/repository/pipelineConfig/DbMigrationConfig.go +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2020 Devtron Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package pipelineConfig - -import ( - "github.com/devtron-labs/devtron/internal/sql/repository" - "github.com/devtron-labs/devtron/pkg/sql" - "github.com/go-pg/pg" - "go.uber.org/zap" -) - -type MigrationTool string - -const MIGRATION_TOOL_MIGRATE MigrationTool = "migrate" - -func (t MigrationTool) IsValid() bool { - types := map[string]MigrationTool{"migrate": MIGRATION_TOOL_MIGRATE} - _, ok := types[string(t)] - return ok -} - -type DbMigrationConfig struct { - tableName struct{} `sql:"db_migration_config" pg:",discard_unknown_columns"` - Id int `sql:"id"` - DbConfigId int `sql:"db_config_id"` - PipelineId int `sql:"pipeline_id"` - GitMaterialId int `sql:"git_material_id"` - ScriptSource string `sql:"script_source"` //location of file in git. relative to git root - MigrationTool MigrationTool `sql:"migration_tool"` - Active bool `sql:"active"` - sql.AuditLog - DbConfig *repository.DbConfig - GitMaterial *GitMaterial -} - -type DbMigrationConfigRepository interface { - Save(config *DbMigrationConfig) error - FindByPipelineId(pipelineId int) (config *DbMigrationConfig, err error) - Update(config *DbMigrationConfig) error -} - -type DbMigrationConfigRepositoryImpl struct { - dbConnection *pg.DB - logger *zap.SugaredLogger -} - -func NewDbMigrationConfigRepositoryImpl(dbConnection *pg.DB, logger *zap.SugaredLogger) *DbMigrationConfigRepositoryImpl { - return &DbMigrationConfigRepositoryImpl{ - dbConnection: dbConnection, - logger: logger, - } -} - -func (impl DbMigrationConfigRepositoryImpl) Save(config *DbMigrationConfig) error { - return impl.dbConnection.Insert(config) -} -func (impl DbMigrationConfigRepositoryImpl) Update(config *DbMigrationConfig) error { - _, err := impl.dbConnection.Model(config).WherePK().UpdateNotNull() - return err -} - -func (impl DbMigrationConfigRepositoryImpl) FindByPipelineId(pipelineId int) (config *DbMigrationConfig, err error) { - config = &DbMigrationConfig{} - err = impl.dbConnection.Model(config). - Column("db_migration_config.*", "DbConfig", "GitMaterial", "GitMaterial.GitProvider"). - Where("db_migration_config.pipeline_id =?", pipelineId). - Where("db_migration_config.active =? ", true). - Select() - return config, err -} diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index 29e9c8e826..05c7ddf332 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -111,16 +111,15 @@ type AppServiceImpl struct { ciArtifactRepository repository.CiArtifactRepository pipelineRepository pipelineConfig.PipelineRepository gitFactory *GitFactory - dbMigrationConfigRepository pipelineConfig.DbMigrationConfigRepository eventClient client.EventClient eventFactory client.EventFactory acdClient application.ServiceClient tokenCache *util3.TokenCache acdAuthConfig *util3.ACDAuthConfig enforcer casbin.Enforcer - enforcerUtil rbac.EnforcerUtil - user user.UserService - appListingRepository repository.AppListingRepository + enforcerUtil rbac.EnforcerUtil + user user.UserService + appListingRepository repository.AppListingRepository appRepository app.AppRepository envRepository repository2.EnvironmentRepository pipelineConfigRepository chartConfig.PipelineConfigRepository @@ -201,7 +200,6 @@ func NewAppService( logger *zap.SugaredLogger, ciArtifactRepository repository.CiArtifactRepository, pipelineRepository pipelineConfig.PipelineRepository, - dbMigrationConfigRepository pipelineConfig.DbMigrationConfigRepository, eventClient client.EventClient, eventFactory client.EventFactory, acdClient application.ServiceClient, cache *util3.TokenCache, authConfig *util3.ACDAuthConfig, @@ -260,7 +258,6 @@ func NewAppService( logger: logger, ciArtifactRepository: ciArtifactRepository, pipelineRepository: pipelineRepository, - dbMigrationConfigRepository: dbMigrationConfigRepository, eventClient: eventClient, eventFactory: eventFactory, acdClient: acdClient, diff --git a/pkg/appStore/deployment/service/InstalledAppService.go b/pkg/appStore/deployment/service/InstalledAppService.go index 47af5f5dc3..58c54eb5a1 100644 --- a/pkg/appStore/deployment/service/InstalledAppService.go +++ b/pkg/appStore/deployment/service/InstalledAppService.go @@ -124,9 +124,9 @@ type InstalledAppServiceImpl struct { ArgoK8sClient argocdServer.ArgoK8sClient gitFactory *util.GitFactory aCDAuthConfig *util2.ACDAuthConfig - gitOpsRepository repository3.GitOpsConfigRepository - userService user.UserService - appStoreDeploymentService AppStoreDeploymentService + gitOpsRepository repository3.GitOpsConfigRepository + userService user.UserService + appStoreDeploymentService AppStoreDeploymentService appStoreDeploymentFullModeService appStoreDeploymentFullMode.AppStoreDeploymentFullModeService installedAppRepositoryHistory repository2.InstalledAppVersionHistoryRepository argoUserService argo.ArgoUserService diff --git a/pkg/appStore/deployment/service/InstalledAppService_test.go b/pkg/appStore/deployment/service/InstalledAppService_test.go index 3f8bb5d4de..7de0bb6574 100644 --- a/pkg/appStore/deployment/service/InstalledAppService_test.go +++ b/pkg/appStore/deployment/service/InstalledAppService_test.go @@ -43,9 +43,9 @@ func TestInstalledAppServiceImpl_DeployDefaultChartOnCluster(t *testing.T) { ArgoK8sClient argocdServer.ArgoK8sClient gitFactory *util.GitFactory aCDAuthConfig *util2.ACDAuthConfig - gitOpsRepository repository3.GitOpsConfigRepository - userService user.UserService - appStoreDeploymentService AppStoreDeploymentService + gitOpsRepository repository3.GitOpsConfigRepository + userService user.UserService + appStoreDeploymentService AppStoreDeploymentService appStoreDeploymentFullModeService appStoreDeploymentFullMode.AppStoreDeploymentFullModeService } type args struct { diff --git a/pkg/appStore/deployment/tool/gitops/AppStoreDeploymentArgoCdService.go b/pkg/appStore/deployment/tool/gitops/AppStoreDeploymentArgoCdService.go index 143a4e437b..c861291f38 100644 --- a/pkg/appStore/deployment/tool/gitops/AppStoreDeploymentArgoCdService.go +++ b/pkg/appStore/deployment/tool/gitops/AppStoreDeploymentArgoCdService.go @@ -74,9 +74,9 @@ type AppStoreDeploymentArgoCdServiceImpl struct { helmAppService client.HelmAppService gitOpsConfigRepository repository3.GitOpsConfigRepository appStatusService appStatus.AppStatusService - pipelineStatusTimelineService status.PipelineStatusTimelineService - userService user.UserService - pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository + pipelineStatusTimelineService status.PipelineStatusTimelineService + userService user.UserService + pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository argoClientWrapperService argocdServer.ArgoClientWrapperService acdConfig *argocdServer.ACDConfig diff --git a/pkg/auth/user/bean/bean.go b/pkg/auth/user/bean/bean.go index 93202f386e..b4d7ea9b1a 100644 --- a/pkg/auth/user/bean/bean.go +++ b/pkg/auth/user/bean/bean.go @@ -7,24 +7,24 @@ const ( ENV_TYPE = "environment" APP_TYPE = "app" WorkflowType = "workflow" - CHART_GROUP_TYPE = "chart-group" - MANAGER_TYPE RoleType = "manager" - ADMIN_TYPE RoleType = "admin" - TRIGGER_TYPE RoleType = "trigger" - VIEW_TYPE RoleType = "view" - ENTITY_ALL_TYPE RoleType = "entityAll" - ENTITY_VIEW_TYPE RoleType = "entityView" - ENTITY_SPECIFIC_TYPE RoleType = "entitySpecific" - ENTITY_SPECIFIC_ADMIN_TYPE RoleType = "entitySpecificAdmin" - ENTITY_SPECIFIC_VIEW_TYPE RoleType = "entitySpecificView" - ROLE_SPECIFIC_TYPE RoleType = "roleSpecific" - ENTITY_CLUSTER_ADMIN_TYPE RoleType = "clusterAdmin" - ENTITY_CLUSTER_VIEW_TYPE RoleType = "clusterView" - ADMIN_HELM_TYPE RoleType = "admin" - EDIT_HELM_TYPE RoleType = "edit" - VIEW_HELM_TYPE RoleType = "view" - ENTITY_CLUSTER_EDIT_TYPE RoleType = "clusterEdit" - DEVTRON_APP = "devtron-app" + CHART_GROUP_TYPE = "chart-group" + MANAGER_TYPE RoleType = "manager" + ADMIN_TYPE RoleType = "admin" + TRIGGER_TYPE RoleType = "trigger" + VIEW_TYPE RoleType = "view" + ENTITY_ALL_TYPE RoleType = "entityAll" + ENTITY_VIEW_TYPE RoleType = "entityView" + ENTITY_SPECIFIC_TYPE RoleType = "entitySpecific" + ENTITY_SPECIFIC_ADMIN_TYPE RoleType = "entitySpecificAdmin" + ENTITY_SPECIFIC_VIEW_TYPE RoleType = "entitySpecificView" + ROLE_SPECIFIC_TYPE RoleType = "roleSpecific" + ENTITY_CLUSTER_ADMIN_TYPE RoleType = "clusterAdmin" + ENTITY_CLUSTER_VIEW_TYPE RoleType = "clusterView" + ADMIN_HELM_TYPE RoleType = "admin" + EDIT_HELM_TYPE RoleType = "edit" + VIEW_HELM_TYPE RoleType = "view" + ENTITY_CLUSTER_EDIT_TYPE RoleType = "clusterEdit" + DEVTRON_APP = "devtron-app" SUPER_ADMIN = "super-admin" CLUSTER = "cluster" GLOBAL_ENTITY = "globalEntity" diff --git a/pkg/auth/user/repository/UserAuditRepository.go b/pkg/auth/user/repository/UserAuditRepository.go index e82382181f..6568b57144 100644 --- a/pkg/auth/user/repository/UserAuditRepository.go +++ b/pkg/auth/user/repository/UserAuditRepository.go @@ -16,7 +16,7 @@ */ /* - @description: user crud +@description: user crud */ package repository diff --git a/pkg/auth/user/repository/UserRepository.go b/pkg/auth/user/repository/UserRepository.go index 31864907a1..191b296c07 100644 --- a/pkg/auth/user/repository/UserRepository.go +++ b/pkg/auth/user/repository/UserRepository.go @@ -66,8 +66,8 @@ type UserRoleModel struct { TableName struct{} `sql:"user_roles"` Id int `sql:"id,pk"` UserId int32 `sql:"user_id,notnull"` - RoleId int `sql:"role_id,notnull"` - User UserModel + RoleId int `sql:"role_id,notnull"` + User UserModel sql.AuditLog } diff --git a/pkg/jira/JiraService.go b/pkg/jira/JiraService.go deleted file mode 100644 index e65b04a913..0000000000 --- a/pkg/jira/JiraService.go +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Copyright (c) 2020 Devtron Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package jira - -import ( - "errors" - "github.com/devtron-labs/devtron/internal/sql/repository" - jiraUtil "github.com/devtron-labs/devtron/internal/util/JiraUtil" - "github.com/devtron-labs/devtron/pkg/projectManagementService/jira" - "github.com/devtron-labs/devtron/pkg/sql" - "go.uber.org/zap" - "time" -) - -type ProjectManagementService interface { - UpdateJiraStatus(updateIssueBean *UpdateIssueBean, userId int32) (map[string][]string, error) - SaveAccountDetails(jiraConfig *ConfigBean, userId int32) (*repository.JiraAccountDetails, error) -} - -type ProjectManagementServiceImpl struct { - logger *zap.SugaredLogger - accountValidator jira.AccountValidator - jiraAccountService jira.AccountService - jiraAccountRepository repository.JiraAccountRepository -} - -func NewProjectManagementServiceImpl(logger *zap.SugaredLogger, jiraAccountService jira.AccountService, - jiraAccountRepository repository.JiraAccountRepository, accountValidator jira.AccountValidator) *ProjectManagementServiceImpl { - return &ProjectManagementServiceImpl{ - logger: logger, - jiraAccountService: jiraAccountService, - jiraAccountRepository: jiraAccountRepository, - accountValidator: accountValidator, - } -} - -type Commit struct { - CommitId string `json:"commitId"` - CommitMessage string `json:"commitMessage"` -} - -type UpdateIssueBean struct { - PipelineId int32 `json:"pipelineId" validate:"number,required"` - PipelineStage string `json:"pipelineStage" validate:"required"` - Commits []Commit `json:"commits" validate:"required" ` -} - -type ConfigBean struct { - UserId int32 `json:"userId" validate:"number,required"` - PipelineId int32 `json:"pipelineId" validate:"number,required"` - PipelineStage string `json:"pipelineStage" validate:"required"` - FinalIssueStatus string `json:"finalIssueStatus" validate:"required"` - ProjectManagementToolAuthToken string `json:"projectManagementToolAuthToken" validate:"required"` - CommitIdRegex string `json:"commitIdRegex" validate:"required"` - ToolUserName string `json:"toolUserName" validate:"required"` - CompanyToolUrl string `json:"companyToolUrl" validate:"required"` -} - -type AccountDetails struct { - UserId int32 - ProjectManagementToolAuthToken string - CommitIdRegex string - UserName string - URL string - FinalIssueStatus string -} - -func (impl *ProjectManagementServiceImpl) SaveAccountDetails(jiraConfig *ConfigBean, userId int32) (*repository.JiraAccountDetails, error) { - if jiraConfig == nil { - impl.logger.Errorw("error in saving account details", "jiraConfig", jiraConfig) - return nil, errors.New("failed SaveJiraAccountDetails, invalid accountDetails") - } - - isAuthenticated, err := impl.accountValidator.ValidateUserAccount(jiraConfig.CompanyToolUrl, jiraConfig.ToolUserName, jiraConfig.ProjectManagementToolAuthToken) - if err != nil { - impl.logger.Errorw("some error in saving account details", "err", err) - return nil, err - } - if !isAuthenticated { - impl.logger.Errorw("cannot SaveJiraAccountDetails, jira authentication failed", "isAuthenticated", isAuthenticated) - return nil, errors.New("jira authentication failed") - } - - account := &repository.JiraAccountDetails{ - UserName: jiraConfig.ToolUserName, - AccountURL: jiraConfig.CompanyToolUrl, - AuthToken: jiraConfig.ProjectManagementToolAuthToken, - CommitMessageRegex: jiraConfig.CommitIdRegex, - FinalIssueStatus: jiraConfig.FinalIssueStatus, - PipelineStage: jiraConfig.PipelineStage, - PipelineId: jiraConfig.PipelineId, - AuditLog: sql.AuditLog{ - CreatedBy: jiraConfig.UserId, - CreatedOn: time.Now(), - UpdatedOn: time.Now(), - UpdatedBy: jiraConfig.UserId, - }, - } - - err = impl.jiraAccountRepository.Save(account) - if err != nil { - return nil, err - } - return account, nil -} - -func (impl *ProjectManagementServiceImpl) UpdateJiraStatus(UpdateIssueBean *UpdateIssueBean, userId int32) (map[string][]string, error) { - commits := UpdateIssueBean.Commits - if len(commits) == 0 { - impl.logger.Errorw("no commits provided", "commits len", len(commits)) - return nil, errors.New("no commits provided") - } - accountDetails, err := impl.jiraAccountRepository.FindByPipelineIdAndStage(UpdateIssueBean.PipelineId, UpdateIssueBean.PipelineStage) - if err != nil { - impl.logger.Errorw("failed to get user account details", "err", err) - return nil, err - } - - regex := accountDetails.CommitMessageRegex - commitMap, invalidRegexCommits := impl.buildCommitMap(commits, regex) - - finalIssueStatus := accountDetails.FinalIssueStatus - - updateRequest := &jira.UpdateRequest{ - UserId: userId, - UserName: accountDetails.UserName, - AuthToken: accountDetails.AuthToken, - AccountURL: accountDetails.AccountURL, - CommitMap: commitMap, - } - resp, err := impl.jiraAccountService.UpdateJiraStatus(updateRequest, finalIssueStatus) - if err != nil { - return nil, err - } - invalidCommits := map[string][]string{"invalidCommits": resp} - if invalidRegexCommits != nil { - invalidCommits["invalidRegexCommits"] = invalidRegexCommits - } - return invalidCommits, nil -} - -func (impl *ProjectManagementServiceImpl) buildCommitMap(commits []Commit, regex string) (map[string]string, []string) { - var invalidCommits []string - commitsMap := make(map[string]string) - for _, commit := range commits { - issueIds, err := jiraUtil.ExtractRegex(regex, commit.CommitMessage) - if err != nil { - impl.logger.Errorw("failed to extract issueIds for commit: ", commit.CommitId, "err", err) - invalidCommits = append(invalidCommits, "failed to extract regex for commitId: "+commit.CommitId+" and commit msg: "+commit.CommitMessage) - continue - } - for _, issueId := range issueIds { - if commitsMap[issueId] == "" { - commitsMap[issueId] = commit.CommitId - } - } - } - return commitsMap, invalidCommits -} diff --git a/pkg/notifier/NotificationConfigService.go b/pkg/notifier/NotificationConfigService.go index 43ca1f07e2..123caa3c09 100644 --- a/pkg/notifier/NotificationConfigService.go +++ b/pkg/notifier/NotificationConfigService.go @@ -57,9 +57,9 @@ type NotificationConfigServiceImpl struct { smtpRepository repository.SMTPNotificationRepository teamRepository repository2.TeamRepository environmentRepository repository3.EnvironmentRepository - appRepository app.AppRepository - userRepository repository4.UserRepository - ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository + appRepository app.AppRepository + userRepository repository4.UserRepository + ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository } type NotificationSettingRequest struct { diff --git a/pkg/pipeline/AppArtifactManager.go b/pkg/pipeline/AppArtifactManager.go index 69dfbf762d..7ec6f4d57d 100644 --- a/pkg/pipeline/AppArtifactManager.go +++ b/pkg/pipeline/AppArtifactManager.go @@ -52,9 +52,9 @@ type AppArtifactManager interface { type AppArtifactManagerImpl struct { logger *zap.SugaredLogger - cdWorkflowRepository pipelineConfig.CdWorkflowRepository - userService user.UserService - imageTaggingService ImageTaggingService + cdWorkflowRepository pipelineConfig.CdWorkflowRepository + userService user.UserService + imageTaggingService ImageTaggingService ciArtifactRepository repository.CiArtifactRepository ciWorkflowRepository pipelineConfig.CiWorkflowRepository pipelineStageService PipelineStageService diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 2ab9eff647..9ea10e3a33 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -18,10 +18,8 @@ package pipeline import ( - "archive/zip" "bufio" "context" - "encoding/json" "errors" "fmt" "io/ioutil" @@ -84,7 +82,6 @@ type CiHandler interface { FetchCiStatusForTriggerViewV1(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) RefreshMaterialByCiPipelineMaterialId(gitMaterialId int) (refreshRes *gitSensor.RefreshGitMaterialResponse, err error) FetchMaterialInfoByArtifactId(ciArtifactId int, envId int) (*types.GitTriggerInfoResponse, error) - WriteToCreateTestSuites(pipelineId int, buildId int, triggeredBy int) UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild int) error FetchCiStatusForTriggerViewForEnvironment(request resourceGroup.ResourceGroupingRequest, token string) ([]*pipelineConfig.CiWorkflowStatus, error) } @@ -1195,8 +1192,6 @@ func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus } else { impl.Logger.Infof("Step failed notification received for wfID %d with message %s", savedWorkflow.Id, savedWorkflow.Message) } - - impl.WriteToCreateTestSuites(savedWorkflow.CiPipelineId, workflowId, int(savedWorkflow.TriggeredBy)) } } return savedWorkflow.Id, nil @@ -1584,76 +1579,6 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId return gitTriggerInfoResponse, nil } -func (impl *CiHandlerImpl) WriteToCreateTestSuites(pipelineId int, buildId int, triggeredBy int) { - testReportFile, err := impl.DownloadCiWorkflowArtifacts(pipelineId, buildId) - if err != nil { - impl.Logger.Errorw("WriteTestSuite, error in fetching report file from s3", "err", err, "pipelineId", pipelineId, "buildId", buildId) - return - } - if testReportFile == nil { - return - } - read, err := zip.OpenReader(testReportFile.Name()) - if err != nil { - impl.Logger.Errorw("WriteTestSuite, error while open reader", "name", testReportFile.Name()) - return - } - defer read.Close() - const CreatedBy = "created_by" - const TriggerId = "trigger_id" - const CiPipelineId = "ci_pipeline_id" - const XML = "xml" - payload := make(map[string]interface{}) - var reports []string - payload[CreatedBy] = triggeredBy - payload[TriggerId] = buildId - payload[CiPipelineId] = pipelineId - payload[XML] = reports - for _, file := range read.File { - if payload, err = impl.listFiles(file, payload); err != nil { - impl.Logger.Errorw("WriteTestSuite, failed to read from zip", "file", file.Name, "error", err) - return - } - } - b, err := json.Marshal(payload) - if err != nil { - impl.Logger.Errorw("WriteTestSuite, payload marshal error", "error", err) - return - } - impl.Logger.Debugw("WriteTestSuite, sending to create", "TriggerId", buildId) - _, err = impl.eventClient.SendTestSuite(b) - if err != nil { - impl.Logger.Errorw("WriteTestSuite, error while making test suit post request", "err", err) - return - } -} - -func (impl *CiHandlerImpl) listFiles(file *zip.File, payload map[string]interface{}) (map[string]interface{}, error) { - fileRead, err := file.Open() - if err != nil { - return payload, err - } - defer fileRead.Close() - - if strings.Contains(file.Name, ".xml") { - content, err := ioutil.ReadAll(fileRead) - if err != nil { - impl.Logger.Errorw("panic error", "err", err) - return payload, err - } - var reports []string - if _, ok := payload["xml"]; !ok { - reports = append(reports, string([]byte(content))) - payload["xml"] = reports - } else { - reports = payload["xml"].([]string) - reports = append(reports, string([]byte(content))) - payload["xml"] = reports - } - } - return payload, nil -} - func (impl *CiHandlerImpl) UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild int) error { ciWorkflows, err := impl.ciWorkflowRepository.FindByStatusesIn([]string{Starting, Running}) if err != nil { diff --git a/pkg/pipeline/DbConfigService.go b/pkg/pipeline/DbConfigService.go deleted file mode 100644 index cddc525548..0000000000 --- a/pkg/pipeline/DbConfigService.go +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright (c) 2020 Devtron Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package pipeline - -import ( - "fmt" - "github.com/devtron-labs/devtron/internal/sql/repository" - "github.com/devtron-labs/devtron/pkg/pipeline/types" - "github.com/devtron-labs/devtron/pkg/sql" - "go.uber.org/zap" - "time" -) - -type DbConfigService interface { - Save(dbConfigBean *types.DbConfigBean) (dbConfig *types.DbConfigBean, err error) - GetAll() (dbConfigs []*types.DbConfigBean, err error) - GetById(id int) (dbConfig *types.DbConfigBean, err error) - Update(dbConfigBean *types.DbConfigBean) (dbConfig *types.DbConfigBean, err error) - GetForAutocomplete() (dbConfigs []*types.DbConfigBean, err error) -} -type DbConfigServiceImpl struct { - configRepo repository.DbConfigRepository - logger *zap.SugaredLogger -} - -func NewDbConfigService(configRepo repository.DbConfigRepository, - logger *zap.SugaredLogger) *DbConfigServiceImpl { - return &DbConfigServiceImpl{ - configRepo: configRepo, - logger: logger, - } -} -func (impl DbConfigServiceImpl) Save(dbConfigBean *types.DbConfigBean) (dbConfig *types.DbConfigBean, err error) { - t := repository.DbType(dbConfigBean.Type) - if valid := t.IsValid(); !valid { - impl.logger.Errorw("invalid type", "dbType", dbConfigBean.Type) - return nil, fmt.Errorf("invalid type %s ", dbConfigBean.Type) - } - config := &repository.DbConfig{ - Password: dbConfigBean.Password, - Port: dbConfigBean.Port, - Host: dbConfigBean.Host, - UserName: dbConfigBean.UserName, - Type: t, - Name: dbConfigBean.Name, - Active: dbConfigBean.Active, - DbName: dbConfigBean.DbName, - AuditLog: sql.AuditLog{ - CreatedBy: dbConfigBean.UserId, - UpdatedBy: dbConfigBean.UserId, - CreatedOn: time.Now(), - UpdatedOn: time.Now(), - }, - } - err = impl.configRepo.Save(config) - if err != nil { - impl.logger.Errorw("error in saving db config", "err", err) - return nil, err - } - dbConfigBean.Id = config.Id - return dbConfigBean, nil -} - -func (impl DbConfigServiceImpl) GetAll() (dbConfigs []*types.DbConfigBean, err error) { - configs, err := impl.configRepo.GetAll() - if err != nil { - return nil, err - } - for _, cfg := range configs { - bean := impl.modelToBeanAdaptor(cfg) - dbConfigs = append(dbConfigs, bean) - } - return dbConfigs, err -} -func (impl DbConfigServiceImpl) GetById(id int) (dbConfig *types.DbConfigBean, err error) { - cfg, err := impl.configRepo.GetById(id) - if err != nil { - return nil, err - } - dbConfig = impl.modelToBeanAdaptor(cfg) - return dbConfig, nil -} - -func (impl DbConfigServiceImpl) Update(dbConfigBean *types.DbConfigBean) (dbConfig *types.DbConfigBean, err error) { - var t repository.DbType - if dbConfigBean.Type != "" { - t = repository.DbType(dbConfigBean.Type) - if valid := t.IsValid(); !valid { - impl.logger.Errorw("invalid type", "dbType", dbConfigBean.Type) - return nil, fmt.Errorf("invalid type %s ", dbConfigBean.Type) - } - } - - config := &repository.DbConfig{ - Id: dbConfigBean.Id, - Password: dbConfigBean.Password, - Port: dbConfigBean.Port, - Host: dbConfigBean.Host, - UserName: dbConfigBean.UserName, - Type: t, - Name: dbConfigBean.Name, - Active: dbConfigBean.Active, - DbName: dbConfigBean.DbName, - AuditLog: sql.AuditLog{ - UpdatedBy: dbConfigBean.UserId, - UpdatedOn: time.Now(), - }, - } - _, err = impl.configRepo.Update(config) - return dbConfigBean, err -} - -func (impl DbConfigServiceImpl) modelToBeanAdaptor(conf *repository.DbConfig) (bean *types.DbConfigBean) { - bean = &types.DbConfigBean{ - DbName: conf.DbName, - Active: conf.Active, - Name: conf.Name, - Type: string(conf.Type), - UserName: conf.UserName, - Host: conf.Host, - Port: conf.Port, - Password: conf.Password, - Id: conf.Id, - } - return bean -} - -func (impl DbConfigServiceImpl) GetForAutocomplete() (dbConfigs []*types.DbConfigBean, err error) { - dbConf, err := impl.configRepo.GetActiveForAutocomplete() - if err != nil { - return nil, err - } - for _, cfg := range dbConf { - bean := impl.modelToBeanAdaptor(cfg) - dbConfigs = append(dbConfigs, bean) - } - return dbConfigs, nil -} diff --git a/pkg/pipeline/DbMigrationService.go b/pkg/pipeline/DbMigrationService.go deleted file mode 100644 index cbe6007ba2..0000000000 --- a/pkg/pipeline/DbMigrationService.go +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright (c) 2020 Devtron Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package pipeline - -import ( - "fmt" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/devtron-labs/devtron/pkg/pipeline/types" - "github.com/devtron-labs/devtron/pkg/sql" - "go.uber.org/zap" - "time" -) - -type DbMigrationService interface { - Save(bean *types.DbMigrationConfigBean) (*types.DbMigrationConfigBean, error) - Update(bean *types.DbMigrationConfigBean) (*types.DbMigrationConfigBean, error) - GetByPipelineId(pipelineId int) (*types.DbMigrationConfigBean, error) -} -type DbMigrationServiceImpl struct { - logger *zap.SugaredLogger - dbMigrationConfigRepository pipelineConfig.DbMigrationConfigRepository -} - -func NewDbMogrationService(logger *zap.SugaredLogger, - dbMigrationConfigRepository pipelineConfig.DbMigrationConfigRepository) *DbMigrationServiceImpl { - return &DbMigrationServiceImpl{ - dbMigrationConfigRepository: dbMigrationConfigRepository, - logger: logger, - } -} - -func (impl DbMigrationServiceImpl) Save(bean *types.DbMigrationConfigBean) (*types.DbMigrationConfigBean, error) { - if valid := pipelineConfig.MigrationTool(bean.MigrationTool).IsValid(); !valid { - return nil, fmt.Errorf("unsupported migration tool %s", bean.MigrationTool) - } - migrationConfig := impl.beanToModelAdaptor(bean) - migrationConfig.AuditLog = sql.AuditLog{ - UpdatedOn: time.Now(), - CreatedOn: time.Now(), - CreatedBy: bean.UserId, - UpdatedBy: bean.UserId, - } - err := impl.dbMigrationConfigRepository.Save(migrationConfig) - if err != nil { - impl.logger.Errorw("error in saving db migration config", "cfg", bean, "err", err) - return nil, err - } - bean.Id = migrationConfig.Id - return bean, nil -} - -func (impl DbMigrationServiceImpl) Update(bean *types.DbMigrationConfigBean) (*types.DbMigrationConfigBean, error) { - if bean.MigrationTool != "" { - if valid := pipelineConfig.MigrationTool(bean.MigrationTool).IsValid(); !valid { - return nil, fmt.Errorf("unsupported migration tool %s", bean.MigrationTool) - } - } - - migrationConfig := impl.beanToModelAdaptor(bean) - migrationConfig.AuditLog = sql.AuditLog{ - UpdatedOn: time.Now(), - UpdatedBy: bean.UserId, - } - err := impl.dbMigrationConfigRepository.Update(migrationConfig) - if err != nil { - impl.logger.Errorw("error in updating db migration config", "cfg", bean, "err", err) - return nil, err - } - bean.Id = migrationConfig.Id - return bean, nil -} - -func (impl DbMigrationServiceImpl) GetByPipelineId(pipelineId int) (*types.DbMigrationConfigBean, error) { - cfg, err := impl.dbMigrationConfigRepository.FindByPipelineId(pipelineId) - if err != nil { - impl.logger.Errorw("error in fetching pipeline db migration config", "id", pipelineId, "err", err) - return nil, err - } - bean := &types.DbMigrationConfigBean{ - MigrationTool: string(cfg.MigrationTool), - GitMaterialId: cfg.GitMaterialId, - PipelineId: cfg.PipelineId, - ScriptSource: cfg.ScriptSource, - Active: cfg.Active, - DbConfigId: cfg.DbConfigId, - Id: cfg.Id, - } - return bean, nil - -} - -func (impl DbMigrationServiceImpl) beanToModelAdaptor(bean *types.DbMigrationConfigBean) *pipelineConfig.DbMigrationConfig { - - migrationConfig := &pipelineConfig.DbMigrationConfig{ - Id: bean.Id, - DbConfigId: bean.DbConfigId, - Active: bean.Active, - ScriptSource: bean.ScriptSource, - PipelineId: bean.PipelineId, - GitMaterialId: bean.GitMaterialId, - MigrationTool: pipelineConfig.MigrationTool(bean.MigrationTool), - } - return migrationConfig -} diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index 7a8ac4a6db..a4c53eca50 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -132,8 +132,6 @@ type WorkflowDagExecutorImpl struct { enforcer casbin.Enforcer enforcerUtil rbac.EnforcerUtil groupRepository repository.DeploymentGroupRepository - tokenCache *util3.TokenCache - acdAuthConfig *util3.ACDAuthConfig envRepository repository2.EnvironmentRepository eventFactory client.EventFactory eventClient client.EventClient @@ -155,9 +153,8 @@ type WorkflowDagExecutorImpl struct { appServiceConfig *app.AppServiceConfig globalPluginService plugin.GlobalPluginService - scopedVariableManager variables.ScopedVariableCMCSManager - variableSnapshotHistoryService variables.VariableSnapshotHistoryService - pluginInputVariableParser PluginInputVariableParser + scopedVariableManager variables.ScopedVariableCMCSManager + pluginInputVariableParser PluginInputVariableParser devtronAsyncHelmInstallRequestMap map[int]bool devtronAsyncHelmInstallRequestLock *sync.Mutex @@ -190,7 +187,6 @@ type WorkflowDagExecutorImpl struct { environmentConfigRepository chartConfig.EnvConfigOverrideRepository appLevelMetricsRepository repository.AppLevelMetricsRepository envLevelMetricsRepository repository.EnvLevelAppMetricsRepository - dbMigrationConfigRepository pipelineConfig.DbMigrationConfigRepository mergeUtil *util.MergeUtil gitOpsConfigRepository repository.GitOpsConfigRepository gitFactory *util.GitFactory @@ -254,8 +250,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi user user.UserService, groupRepository repository.DeploymentGroupRepository, envRepository repository2.EnvironmentRepository, - enforcer casbin.Enforcer, enforcerUtil rbac.EnforcerUtil, tokenCache *util3.TokenCache, - acdAuthConfig *util3.ACDAuthConfig, eventFactory client.EventFactory, + enforcer casbin.Enforcer, enforcerUtil rbac.EnforcerUtil, eventFactory client.EventFactory, eventClient client.EventClient, cvePolicyRepository security.CvePolicyRepository, scanResultRepository security.ImageScanResultRepository, appWorkflowRepository appWorkflow.AppWorkflowRepository, @@ -267,7 +262,6 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi ciWorkflowRepository pipelineConfig.CiWorkflowRepository, appLabelRepository pipelineConfig.AppLabelRepository, gitSensorGrpcClient gitSensorClient.Client, pipelineStageService PipelineStageService, k8sCommonService k8s.K8sCommonService, - variableSnapshotHistoryService variables.VariableSnapshotHistoryService, globalPluginService plugin.GlobalPluginService, pluginInputVariableParser PluginInputVariableParser, scopedVariableManager variables.ScopedVariableCMCSManager, @@ -298,7 +292,6 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi environmentConfigRepository chartConfig.EnvConfigOverrideRepository, appLevelMetricsRepository repository.AppLevelMetricsRepository, envLevelMetricsRepository repository.EnvLevelAppMetricsRepository, - dbMigrationConfigRepository pipelineConfig.DbMigrationConfigRepository, mergeUtil *util.MergeUtil, gitOpsConfigRepository repository.GitOpsConfigRepository, gitFactory *util.GitFactory, @@ -322,8 +315,6 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi enforcer: enforcer, enforcerUtil: enforcerUtil, groupRepository: groupRepository, - tokenCache: tokenCache, - acdAuthConfig: acdAuthConfig, envRepository: envRepository, eventFactory: eventFactory, eventClient: eventClient, @@ -375,7 +366,6 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi environmentConfigRepository: environmentConfigRepository, appLevelMetricsRepository: appLevelMetricsRepository, envLevelMetricsRepository: envLevelMetricsRepository, - dbMigrationConfigRepository: dbMigrationConfigRepository, mergeUtil: mergeUtil, gitOpsConfigRepository: gitOpsConfigRepository, gitFactory: gitFactory, @@ -3256,8 +3246,8 @@ func (impl *WorkflowDagExecutorImpl) GetValuesOverrideForTrigger(overrideRequest return valuesOverrideResponse, err } var ( - pipelineOverride *chartConfig.PipelineOverride - dbMigrationOverride, configMapJson, appLabelJsonByte []byte + pipelineOverride *chartConfig.PipelineOverride + configMapJson, appLabelJsonByte []byte ) // Conditional Block based on PipelineOverrideCreated --> start @@ -3288,15 +3278,6 @@ func (impl *WorkflowDagExecutorImpl) GetValuesOverrideForTrigger(overrideRequest // Conditional Block based on PipelineOverrideCreated --> start if !isPipelineOverrideCreated { - _, span = otel.Tracer("orchestrator").Start(ctx, "getDbMigrationOverride") - //FIXME: how to determine rollback - //we can't depend on ciArtifact ID because CI pipeline can be manually triggered in any order regardless of sourcecode status - dbMigrationOverride, err = impl.getDbMigrationOverride(overrideRequest, artifact, false) - span.End() - if err != nil { - impl.logger.Errorw("error in fetching db migration config", "req", overrideRequest, "err", err) - return valuesOverrideResponse, err - } chartVersion := envOverride.Chart.ChartVersion _, span = otel.Tracer("orchestrator").Start(ctx, "getConfigMapAndSecretJsonV2") scope := getScopeForVariables(overrideRequest, envOverride) @@ -3316,7 +3297,7 @@ func (impl *WorkflowDagExecutorImpl) GetValuesOverrideForTrigger(overrideRequest appLabelJsonByte = nil } - mergedValues, err := impl.mergeOverrideValues(envOverride, dbMigrationOverride, releaseOverrideJson, configMapJson, appLabelJsonByte, strategy) + mergedValues, err := impl.mergeOverrideValues(envOverride, releaseOverrideJson, configMapJson, appLabelJsonByte, strategy) appName := fmt.Sprintf("%s-%s", overrideRequest.AppName, envOverride.Environment.Name) mergedValues = impl.autoscalingCheckBeforeTrigger(ctx, appName, envOverride.Namespace, mergedValues, overrideRequest) @@ -3838,70 +3819,6 @@ func (impl *WorkflowDagExecutorImpl) GetAppMetricsByTriggerType(overrideRequest return appMetrics, nil } -func (impl *WorkflowDagExecutorImpl) getDbMigrationOverride(overrideRequest *bean.ValuesOverrideRequest, artifact *repository.CiArtifact, isRollback bool) (overrideJson []byte, err error) { - if isRollback { - return nil, fmt.Errorf("rollback not supported ye") - } - notConfigured := false - config, err := impl.dbMigrationConfigRepository.FindByPipelineId(overrideRequest.PipelineId) - if err != nil && !util.IsErrNoRows(err) { - impl.logger.Errorw("error in fetching pipeline override config", "req", overrideRequest, "err", err) - return nil, err - } else if util.IsErrNoRows(err) { - notConfigured = true - } - envVal := &EnvironmentOverride{} - if notConfigured { - impl.logger.Warnw("no active db migration found", "pipeline", overrideRequest.PipelineId) - envVal.Enabled = false - } else { - materialInfos, err := artifact.ParseMaterialInfo() - if err != nil { - return nil, err - } - - hash, ok := materialInfos[config.GitMaterial.Url] - if !ok { - impl.logger.Errorf("wrong url map ", "map", materialInfos, "url", config.GitMaterial.Url) - return nil, fmt.Errorf("configured url not found in material %s", config.GitMaterial.Url) - } - - envVal.Enabled = true - if config.GitMaterial.GitProvider.AuthMode != repository.AUTH_MODE_USERNAME_PASSWORD && - config.GitMaterial.GitProvider.AuthMode != repository.AUTH_MODE_ACCESS_TOKEN && - config.GitMaterial.GitProvider.AuthMode != repository.AUTH_MODE_ANONYMOUS { - return nil, fmt.Errorf("auth mode %s not supported for migration", config.GitMaterial.GitProvider.AuthMode) - } - envVal.appendEnvironmentVariable("GIT_REPO_URL", config.GitMaterial.Url) - envVal.appendEnvironmentVariable("GIT_USER", config.GitMaterial.GitProvider.UserName) - var password string - if config.GitMaterial.GitProvider.AuthMode == repository.AUTH_MODE_USERNAME_PASSWORD { - password = config.GitMaterial.GitProvider.Password - } else { - password = config.GitMaterial.GitProvider.AccessToken - } - envVal.appendEnvironmentVariable("GIT_AUTH_TOKEN", password) - // parse git-tag not required - //envVal.appendEnvironmentVariable("GIT_TAG", "") - envVal.appendEnvironmentVariable("GIT_HASH", hash) - envVal.appendEnvironmentVariable("SCRIPT_LOCATION", config.ScriptSource) - envVal.appendEnvironmentVariable("DB_TYPE", string(config.DbConfig.Type)) - envVal.appendEnvironmentVariable("DB_USER_NAME", config.DbConfig.UserName) - envVal.appendEnvironmentVariable("DB_PASSWORD", config.DbConfig.Password) - envVal.appendEnvironmentVariable("DB_HOST", config.DbConfig.Host) - envVal.appendEnvironmentVariable("DB_PORT", config.DbConfig.Port) - envVal.appendEnvironmentVariable("DB_NAME", config.DbConfig.DbName) - //Will be used for rollback don't delete it - //envVal.appendEnvironmentVariable("MIGRATE_TO_VERSION", strconv.Itoa(overrideRequest.TargetDbVersion)) - } - dbMigrationConfig := map[string]interface{}{"dbMigrationConfig": envVal} - confByte, err := json.Marshal(dbMigrationConfig) - if err != nil { - return nil, err - } - return confByte, nil -} - type ConfigMapAndSecretJsonV2 struct { AppId int EnvId int @@ -4095,142 +4012,7 @@ func (impl *WorkflowDagExecutorImpl) getReleaseOverride(envOverride *chartConfig return override, nil } -func (impl *WorkflowDagExecutorImpl) mergeAndSave(envOverride *chartConfig.EnvConfigOverride, - overrideRequest *bean.ValuesOverrideRequest, - dbMigrationOverride []byte, - artifact *repository.CiArtifact, - pipeline *pipelineConfig.Pipeline, configMapJson, appLabelJsonByte []byte, strategy *chartConfig.PipelineStrategy, ctx context.Context, - triggeredAt time.Time, deployedBy int32, appMetrics *bool) (releaseId int, overrideId int, mergedValues string, err error) { - - //register release , obtain release id TODO: populate releaseId to template - override, err := impl.savePipelineOverride(overrideRequest, envOverride.Id, triggeredAt) - if err != nil { - return 0, 0, "", err - } - //TODO: check status and apply lock - overrideJson, err := impl.getReleaseOverride(envOverride, overrideRequest, artifact, override, strategy, appMetrics) - if err != nil { - return 0, 0, "", err - } - - //merge three values on the fly - //ordering is important here - //global < environment < db< release - var merged []byte - if !envOverride.IsOverride { - merged, err = impl.mergeUtil.JsonPatch([]byte("{}"), []byte(envOverride.Chart.GlobalOverride)) - if err != nil { - return 0, 0, "", err - } - } else { - merged, err = impl.mergeUtil.JsonPatch([]byte("{}"), []byte(envOverride.EnvOverrideValues)) - if err != nil { - return 0, 0, "", err - } - } - - //pipeline override here comes from pipeline strategy table - if strategy != nil && len(strategy.Config) > 0 { - merged, err = impl.mergeUtil.JsonPatch(merged, []byte(strategy.Config)) - if err != nil { - return 0, 0, "", err - } - } - merged, err = impl.mergeUtil.JsonPatch(merged, dbMigrationOverride) - if err != nil { - return 0, 0, "", err - } - merged, err = impl.mergeUtil.JsonPatch(merged, []byte(overrideJson)) - if err != nil { - return 0, 0, "", err - } - - if configMapJson != nil { - merged, err = impl.mergeUtil.JsonPatch(merged, configMapJson) - if err != nil { - return 0, 0, "", err - } - } - - if appLabelJsonByte != nil { - merged, err = impl.mergeUtil.JsonPatch(merged, appLabelJsonByte) - if err != nil { - return 0, 0, "", err - } - } - - appName := fmt.Sprintf("%s-%s", pipeline.App.AppName, envOverride.Environment.Name) - merged = impl.autoscalingCheckBeforeTrigger(ctx, appName, envOverride.Namespace, merged, overrideRequest) - - _, span := otel.Tracer("orchestrator").Start(ctx, "dockerRegistryIpsConfigService.HandleImagePullSecretOnApplicationDeployment") - // handle image pull secret if access given - merged, err = impl.dockerRegistryIpsConfigService.HandleImagePullSecretOnApplicationDeployment(envOverride.Environment, artifact, pipeline.CiPipelineId, merged) - span.End() - if err != nil { - return 0, 0, "", err - } - - commitHash := "" - commitTime := time.Time{} - if util.IsAcdApp(pipeline.DeploymentAppType) { - chartRepoName := impl.chartTemplateService.GetGitOpsRepoNameFromUrl(envOverride.Chart.GitRepoUrl) - _, span = otel.Tracer("orchestrator").Start(ctx, "chartTemplateService.GetUserEmailIdAndNameForGitOpsCommit") - //getting username & emailId for commit author data - userEmailId, userName := impl.chartTemplateService.GetUserEmailIdAndNameForGitOpsCommit(overrideRequest.UserId) - span.End() - chartGitAttr := &util.ChartConfig{ - FileName: fmt.Sprintf("_%d-values.yaml", envOverride.TargetEnvironment), - FileContent: string(merged), - ChartName: envOverride.Chart.ChartName, - ChartLocation: envOverride.Chart.ChartLocation, - ChartRepoName: chartRepoName, - ReleaseMessage: fmt.Sprintf("release-%d-env-%d ", override.Id, envOverride.TargetEnvironment), - UserName: userName, - UserEmailId: userEmailId, - } - gitOpsConfigBitbucket, err := impl.gitOpsConfigRepository.GetGitOpsConfigByProvider(util.BITBUCKET_PROVIDER) - if err != nil { - if err == pg.ErrNoRows { - gitOpsConfigBitbucket.BitBucketWorkspaceId = "" - } else { - return 0, 0, "", err - } - } - gitOpsConfig := &bean.GitOpsConfigDto{BitBucketWorkspaceId: gitOpsConfigBitbucket.BitBucketWorkspaceId} - _, span = otel.Tracer("orchestrator").Start(ctx, "gitFactory.Client.CommitValues") - commitHash, commitTime, err = impl.gitFactory.Client.CommitValues(chartGitAttr, gitOpsConfig) - span.End() - if err != nil { - impl.logger.Errorw("error in git commit", "err", err) - return 0, 0, "", err - } - } - if commitTime.IsZero() { - commitTime = time.Now() - } - pipelineOverride := &chartConfig.PipelineOverride{ - Id: override.Id, - GitHash: commitHash, - CommitTime: commitTime, - EnvConfigOverrideId: envOverride.Id, - PipelineOverrideValues: overrideJson, - PipelineId: overrideRequest.PipelineId, - CiArtifactId: overrideRequest.CiArtifactId, - PipelineMergedValues: string(merged), - AuditLog: sql.AuditLog{UpdatedOn: triggeredAt, UpdatedBy: deployedBy}, - } - _, span = otel.Tracer("orchestrator").Start(ctx, "pipelineOverrideRepository.Update") - err = impl.pipelineOverrideRepository.Update(pipelineOverride) - span.End() - if err != nil { - return 0, 0, "", err - } - mergedValues = string(merged) - return override.PipelineReleaseCounter, override.Id, mergedValues, nil -} - func (impl *WorkflowDagExecutorImpl) mergeOverrideValues(envOverride *chartConfig.EnvConfigOverride, - dbMigrationOverride []byte, releaseOverrideJson string, configMapJson []byte, appLabelJsonByte []byte, @@ -4258,10 +4040,6 @@ func (impl *WorkflowDagExecutorImpl) mergeOverrideValues(envOverride *chartConfi return nil, err } } - merged, err = impl.mergeUtil.JsonPatch(merged, dbMigrationOverride) - if err != nil { - return nil, err - } merged, err = impl.mergeUtil.JsonPatch(merged, []byte(releaseOverrideJson)) if err != nil { return nil, err diff --git a/pkg/projectManagementService/jira/JiraAccountValidator.go b/pkg/projectManagementService/jira/JiraAccountValidator.go deleted file mode 100644 index 4ee55ac402..0000000000 --- a/pkg/projectManagementService/jira/JiraAccountValidator.go +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2020 Devtron Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package jira - -import ( - "errors" - "github.com/devtron-labs/devtron/client/jira" - "go.uber.org/zap" -) - -type AccountValidator interface { - ValidateUserAccount(jiraAccountUrl string, userName string, token string) (bool, error) -} - -type AccountValidatorImpl struct { - logger *zap.SugaredLogger - jiraClient client.JiraClient -} - -func NewAccountValidatorImpl(logger *zap.SugaredLogger, jiraClient client.JiraClient) *AccountValidatorImpl { - return &AccountValidatorImpl{logger: logger, jiraClient: jiraClient} -} - -func (impl *AccountValidatorImpl) ValidateUserAccount(jiraAccountUrl string, userName string, token string) (bool, error) { - if jiraAccountUrl == "" || userName == "" || token == "" { - impl.logger.Errorw("cannot validate user account for invalid params", "jiraAccountUrl", jiraAccountUrl, "userName", userName) - return false, errors.New("cannot validate user account for invalid params") - } - - clientReq := client.CreateClientReq(userName, token, jiraAccountUrl+"/test") - resp, err := impl.jiraClient.AuthenticateUserAccount(clientReq) - - if err != nil { - return false, err - } - if resp.StatusCode == 401 { - impl.logger.Errorw("User jira could not be authenticated", "code", resp.StatusCode) - return false, nil - } - return true, nil -} diff --git a/pkg/projectManagementService/jira/JiraManagementService.go b/pkg/projectManagementService/jira/JiraManagementService.go deleted file mode 100644 index f5cc70c27f..0000000000 --- a/pkg/projectManagementService/jira/JiraManagementService.go +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (c) 2020 Devtron Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package jira - -import ( - "errors" - "strings" - - client "github.com/devtron-labs/devtron/client/jira" - "github.com/devtron-labs/devtron/internal/sql/repository" - "go.uber.org/zap" -) - -type AccountService interface { - UpdateJiraStatus(UpdateRequest *UpdateRequest, finalIssueStatus string) ([]string, error) -} - -type UpdateRequest struct { - UserId int32 - UserName string - AuthToken string - AccountURL string - CommitMap map[string]string -} - -type AccountServiceImpl struct { - logger *zap.SugaredLogger - jiraAccountRepository repository.JiraAccountRepository - jiraClient client.JiraClient -} - -func NewAccountServiceImpl(logger *zap.SugaredLogger, jiraAccountRepository repository.JiraAccountRepository, jiraClient client.JiraClient) *AccountServiceImpl { - return &AccountServiceImpl{logger: logger, jiraAccountRepository: jiraAccountRepository, jiraClient: jiraClient} -} - -func (impl *AccountServiceImpl) UpdateJiraStatus(updateRequest *UpdateRequest, finalIssueStatus string) ([]string, error) { - if updateRequest == nil { - return nil, errors.New("cannot UpdateJiraStatus, invalid updateRequest") - } - // TODO: create goroutines - var errorCommits []string - for issueId, commitId := range updateRequest.CommitMap { - clientReq := client.CreateClientReq(updateRequest.UserName, updateRequest.AuthToken, updateRequest.AccountURL) - transitionResponses, err := impl.jiraClient.FindIssueTransitions(clientReq, issueId) - - if err != nil { - impl.logger.Errorw("could not find transitions for ", issueId, "err", err) - errorCommits = impl.handleJiraUpdateError(issueId, err, errorCommits, commitId) - continue - } - - id := "" - for _, jiraTransition := range transitionResponses { - if strings.EqualFold(finalIssueStatus, jiraTransition.Name) { - id = jiraTransition.Id - break - } - } - - if id == "" { - impl.logger.Errorw("no transition id not found for issue:", "issueId", issueId, "and status:", finalIssueStatus, "err", err) - errorCommits = impl.handleJiraUpdateError(issueId, err, errorCommits, commitId) - continue - } - - resp, err := impl.jiraClient.UpdateJiraTransition(clientReq, issueId, id) - if err != nil { - impl.logger.Errorw("could not update transition for ", "issueId", issueId, "err", err) - errorCommits = impl.handleJiraUpdateError(issueId, err, errorCommits, commitId) - continue - } - if resp.StatusCode != 204 { - impl.logger.Errorw("update transition jira response status ", "status", resp.Status) - errorCommits = impl.handleJiraUpdateError(issueId, err, errorCommits, commitId) - } - } - return errorCommits, nil -} - -func (impl *AccountServiceImpl) handleJiraUpdateError(issueId string, err error, errorCommits []string, commitId string) []string { - errorCommits = append(errorCommits, "could not update jira for commitId: "+commitId+" issueId: "+issueId) - return errorCommits -} diff --git a/pkg/security/ImageScanService.go b/pkg/security/ImageScanService.go index d4a8ffbf5e..e8d027e5f1 100644 --- a/pkg/security/ImageScanService.go +++ b/pkg/security/ImageScanService.go @@ -47,9 +47,9 @@ type ImageScanServiceImpl struct { scanResultRepository security.ImageScanResultRepository scanObjectMetaRepository security.ImageScanObjectMetaRepository cveStoreRepository security.CveStoreRepository - imageScanDeployInfoRepository security.ImageScanDeployInfoRepository - userService user.UserService - teamRepository repository2.TeamRepository + imageScanDeployInfoRepository security.ImageScanDeployInfoRepository + userService user.UserService + teamRepository repository2.TeamRepository appRepository repository1.AppRepository envService cluster.EnvironmentService ciArtifactRepository repository.CiArtifactRepository diff --git a/scripts/sql/210_unused_integration.up.sql b/scripts/sql/210_unused_integration.up.sql new file mode 100644 index 0000000000..df31f1b8c6 --- /dev/null +++ b/scripts/sql/210_unused_integration.up.sql @@ -0,0 +1,12 @@ +-- +-- Dropping table: project_management_tool_config; Schema: public; Owner: postgres +-- +DROP TABLE "public"."project_management_tool_config" CASCADE; +-- +-- Dropping table: db_migration_config; Schema: public; Owner: postgres +-- +DROP TABLE "public"."db_migration_config" CASCADE; +-- +-- Dropping table: db_config; Schema: public; Owner: postgres +-- +DROP TABLE "public"."db_config" CASCADE; \ No newline at end of file diff --git a/scripts/sql/210_unused_intergration.down.sql b/scripts/sql/210_unused_intergration.down.sql new file mode 100644 index 0000000000..3b4f014dc9 --- /dev/null +++ b/scripts/sql/210_unused_intergration.down.sql @@ -0,0 +1,125 @@ +CREATE TABLE public.project_management_tool_config ( + id integer NOT NULL, + user_name character varying(250) NOT NULL, + account_url character varying(250) NOT NULL, + auth_token character varying(250) NOT NULL, + commit_message_regex character varying(250) NOT NULL, + final_issue_status character varying(250) NOT NULL, + pipeline_stage character varying(250) NOT NULL, + pipeline_id integer NOT NULL, + created_on timestamp with time zone NOT NULL, + updated_on timestamp with time zone NOT NULL, + created_by integer NOT NULL, + updated_by integer NOT NULL +); + + +ALTER TABLE public.project_management_tool_config OWNER TO postgres; + +-- +-- Name: project_management_tool_config_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE public.project_management_tool_config_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.project_management_tool_config_id_seq OWNER TO postgres; + +-- +-- Name: project_management_tool_config_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE public.project_management_tool_config_id_seq OWNED BY public.project_management_tool_config.id; + +-- +-- Name: db_config; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.db_config ( + id integer NOT NULL, + name character varying(250) NOT NULL, + type character varying(250) NOT NULL, + host character varying(250) NOT NULL, + port character varying(250) NOT NULL, + db_name character varying(250) NOT NULL, + user_name character varying(250) NOT NULL, + password character varying(250) NOT NULL, + active boolean NOT NULL, + created_on timestamp with time zone NOT NULL, + created_by integer NOT NULL, + updated_on timestamp with time zone NOT NULL, + updated_by integer NOT NULL +); + + +ALTER TABLE public.db_config OWNER TO postgres; + +-- +-- Name: db_config_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE public.db_config_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.db_config_id_seq OWNER TO postgres; + +-- +-- Name: db_config_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE public.db_config_id_seq OWNED BY public.db_config.id; + + +-- +-- Name: db_migration_config; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.db_migration_config ( + id integer NOT NULL, + db_config_id integer NOT NULL, + pipeline_id integer NOT NULL, + git_material_id integer NOT NULL, + script_source character varying(250) NOT NULL, + migration_tool character varying(250) NOT NULL, + active boolean NOT NULL, + created_on timestamp with time zone NOT NULL, + created_by integer NOT NULL, + updated_on timestamp with time zone NOT NULL, + updated_by integer NOT NULL +); + + +ALTER TABLE public.db_migration_config OWNER TO postgres; + +-- +-- Name: db_migration_config_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE public.db_migration_config_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE public.db_migration_config_id_seq OWNER TO postgres; + +-- +-- Name: db_migration_config_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres +-- + +ALTER SEQUENCE public.db_migration_config_id_seq OWNED BY public.db_migration_config.id; diff --git a/wire_gen.go b/wire_gen.go index 1ec6b644e1..e871688b17 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -51,7 +51,6 @@ import ( "github.com/devtron-labs/devtron/client/events" "github.com/devtron-labs/devtron/client/gitSensor" "github.com/devtron-labs/devtron/client/grafana" - client4 "github.com/devtron-labs/devtron/client/jira" "github.com/devtron-labs/devtron/client/lens" "github.com/devtron-labs/devtron/client/telemetry" "github.com/devtron-labs/devtron/internal/sql/repository" @@ -112,7 +111,6 @@ import ( repository10 "github.com/devtron-labs/devtron/pkg/genericNotes/repository" "github.com/devtron-labs/devtron/pkg/git" "github.com/devtron-labs/devtron/pkg/gitops" - jira2 "github.com/devtron-labs/devtron/pkg/jira" k8s2 "github.com/devtron-labs/devtron/pkg/k8s" application2 "github.com/devtron-labs/devtron/pkg/k8s/application" "github.com/devtron-labs/devtron/pkg/k8s/capacity" @@ -131,7 +129,6 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/plugin" repository12 "github.com/devtron-labs/devtron/pkg/plugin/repository" - "github.com/devtron-labs/devtron/pkg/projectManagementService/jira" resourceGroup2 "github.com/devtron-labs/devtron/pkg/resourceGroup" "github.com/devtron-labs/devtron/pkg/resourceQualifiers" security2 "github.com/devtron-labs/devtron/pkg/security" @@ -179,7 +176,6 @@ func InitializeApp() (*App, error) { } ciArtifactRepositoryImpl := repository.NewCiArtifactRepositoryImpl(db, sugaredLogger) pipelineRepositoryImpl := pipelineConfig.NewPipelineRepositoryImpl(db, sugaredLogger) - dbMigrationConfigRepositoryImpl := pipelineConfig.NewDbMigrationConfigRepositoryImpl(db, sugaredLogger) httpClient := util.NewHttpClient() eventClientConfig, err := client.GetEventClientConfig() if err != nil { @@ -426,7 +422,7 @@ func InitializeApp() (*App, error) { k8sCommonServiceImpl := k8s2.NewK8sCommonServiceImpl(sugaredLogger, k8sUtil, clusterServiceImplExtended) manifestPushConfigRepositoryImpl := repository11.NewManifestPushConfigRepository(sugaredLogger, db) gitOpsManifestPushServiceImpl := app2.NewGitOpsManifestPushServiceImpl(sugaredLogger, chartTemplateServiceImpl, chartServiceImpl, gitOpsConfigRepositoryImpl, gitFactory, pipelineStatusTimelineServiceImpl, pipelineStatusTimelineRepositoryImpl, acdConfig) - appServiceImpl := app2.NewAppService(envConfigOverrideRepositoryImpl, pipelineOverrideRepositoryImpl, mergeUtil, sugaredLogger, ciArtifactRepositoryImpl, pipelineRepositoryImpl, dbMigrationConfigRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, applicationServiceClientImpl, tokenCache, acdAuthConfig, enforcerImpl, enforcerUtilImpl, userServiceImpl, appListingRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, chartRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, imageScanDeployInfoRepositoryImpl, imageScanHistoryRepositoryImpl, argoK8sClientImpl, gitFactory, pipelineStrategyHistoryServiceImpl, configMapHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, chartTemplateServiceImpl, refChartDir, chartRefRepositoryImpl, chartServiceImpl, helmAppClientImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, appCrudOperationServiceImpl, configMapHistoryRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, dockerRegistryIpsConfigServiceImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, gitOpsConfigRepositoryImpl, appStatusServiceImpl, installedAppRepositoryImpl, appStoreDeploymentServiceImpl, k8sCommonServiceImpl, installedAppVersionHistoryRepositoryImpl, globalEnvVariables, helmAppServiceImpl, manifestPushConfigRepositoryImpl, gitOpsManifestPushServiceImpl, argoClientWrapperServiceImpl, scopedVariableCMCSManagerImpl, acdConfig) + appServiceImpl := app2.NewAppService(envConfigOverrideRepositoryImpl, pipelineOverrideRepositoryImpl, mergeUtil, sugaredLogger, ciArtifactRepositoryImpl, pipelineRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, applicationServiceClientImpl, tokenCache, acdAuthConfig, enforcerImpl, enforcerUtilImpl, userServiceImpl, appListingRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, chartRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, imageScanDeployInfoRepositoryImpl, imageScanHistoryRepositoryImpl, argoK8sClientImpl, gitFactory, pipelineStrategyHistoryServiceImpl, configMapHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, chartTemplateServiceImpl, refChartDir, chartRefRepositoryImpl, chartServiceImpl, helmAppClientImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, appCrudOperationServiceImpl, configMapHistoryRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, dockerRegistryIpsConfigServiceImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, gitOpsConfigRepositoryImpl, appStatusServiceImpl, installedAppRepositoryImpl, appStoreDeploymentServiceImpl, k8sCommonServiceImpl, installedAppVersionHistoryRepositoryImpl, globalEnvVariables, helmAppServiceImpl, manifestPushConfigRepositoryImpl, gitOpsManifestPushServiceImpl, argoClientWrapperServiceImpl, scopedVariableCMCSManagerImpl, acdConfig) validate, err := util.IntValidator() if err != nil { return nil, err @@ -467,7 +463,7 @@ func InitializeApp() (*App, error) { customTagServiceImpl := pipeline.NewCustomTagService(sugaredLogger, imageTagRepositoryImpl) pluginInputVariableParserImpl := pipeline.NewPluginInputVariableParserImpl(sugaredLogger, dockerRegistryConfigImpl, customTagServiceImpl) pipelineConfigListenerServiceImpl := pipeline.NewPipelineConfigListenerServiceImpl(sugaredLogger) - workflowDagExecutorImpl := pipeline.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, pubSubClientServiceImpl, appServiceImpl, workflowServiceImpl, ciArtifactRepositoryImpl, ciPipelineRepositoryImpl, materialRepositoryImpl, pipelineOverrideRepositoryImpl, userServiceImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, enforcerImpl, enforcerUtilImpl, tokenCache, acdAuthConfig, eventSimpleFactoryImpl, eventRESTClientImpl, cvePolicyRepositoryImpl, imageScanResultRepositoryImpl, appWorkflowRepositoryImpl, prePostCdScriptHistoryServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineServiceImpl, ciTemplateRepositoryImpl, ciWorkflowRepositoryImpl, appLabelRepositoryImpl, clientImpl, pipelineStageServiceImpl, k8sCommonServiceImpl, variableSnapshotHistoryServiceImpl, globalPluginServiceImpl, pluginInputVariableParserImpl, scopedVariableCMCSManagerImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineStrategyHistoryServiceImpl, manifestPushConfigRepositoryImpl, gitOpsManifestPushServiceImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, appCrudOperationServiceImpl, pipelineConfigRepositoryImpl, dockerRegistryIpsConfigServiceImpl, chartRepositoryImpl, chartTemplateServiceImpl, pipelineStrategyHistoryRepositoryImpl, appRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, argoK8sClientImpl, configMapRepositoryImpl, configMapHistoryRepositoryImpl, refChartDir, helmAppServiceImpl, helmAppClientImpl, chartRefRepositoryImpl, envConfigOverrideRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, dbMigrationConfigRepositoryImpl, mergeUtil, gitOpsConfigRepositoryImpl, gitFactory, applicationServiceClientImpl, argoClientWrapperServiceImpl, pipelineConfigListenerServiceImpl, customTagServiceImpl, acdConfig) + workflowDagExecutorImpl := pipeline.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, pubSubClientServiceImpl, appServiceImpl, workflowServiceImpl, ciArtifactRepositoryImpl, ciPipelineRepositoryImpl, materialRepositoryImpl, pipelineOverrideRepositoryImpl, userServiceImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, enforcerImpl, enforcerUtilImpl, eventSimpleFactoryImpl, eventRESTClientImpl, cvePolicyRepositoryImpl, imageScanResultRepositoryImpl, appWorkflowRepositoryImpl, prePostCdScriptHistoryServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineServiceImpl, ciTemplateRepositoryImpl, ciWorkflowRepositoryImpl, appLabelRepositoryImpl, clientImpl, pipelineStageServiceImpl, k8sCommonServiceImpl, globalPluginServiceImpl, pluginInputVariableParserImpl, scopedVariableCMCSManagerImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineStrategyHistoryServiceImpl, manifestPushConfigRepositoryImpl, gitOpsManifestPushServiceImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, appCrudOperationServiceImpl, pipelineConfigRepositoryImpl, dockerRegistryIpsConfigServiceImpl, chartRepositoryImpl, chartTemplateServiceImpl, pipelineStrategyHistoryRepositoryImpl, appRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, argoK8sClientImpl, configMapRepositoryImpl, configMapHistoryRepositoryImpl, refChartDir, helmAppServiceImpl, helmAppClientImpl, chartRefRepositoryImpl, envConfigOverrideRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, mergeUtil, gitOpsConfigRepositoryImpl, gitFactory, applicationServiceClientImpl, argoClientWrapperServiceImpl, pipelineConfigListenerServiceImpl, customTagServiceImpl, acdConfig) deploymentGroupAppRepositoryImpl := repository.NewDeploymentGroupAppRepositoryImpl(sugaredLogger, db) deploymentGroupServiceImpl := deploymentGroup.NewDeploymentGroupServiceImpl(appRepositoryImpl, sugaredLogger, pipelineRepositoryImpl, ciPipelineRepositoryImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, deploymentGroupAppRepositoryImpl, ciArtifactRepositoryImpl, appWorkflowRepositoryImpl, workflowDagExecutorImpl) deploymentConfigServiceImpl := pipeline.NewDeploymentConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, pipelineRepositoryImpl, envLevelAppMetricsRepositoryImpl, appLevelMetricsRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, configMapHistoryServiceImpl, chartRefRepositoryImpl, scopedVariableCMCSManagerImpl) @@ -513,7 +509,6 @@ func InitializeApp() (*App, error) { appDeploymentTypeChangeManagerImpl := pipeline.NewAppDeploymentTypeChangeManagerImpl(sugaredLogger, pipelineRepositoryImpl, workflowDagExecutorImpl, appServiceImpl, chartTemplateServiceImpl, appStatusRepositoryImpl, helmAppServiceImpl, applicationServiceClientImpl, appArtifactManagerImpl, cdPipelineConfigServiceImpl) devtronAppConfigServiceImpl := pipeline.NewDevtronAppConfigServiceImpl(sugaredLogger, ciCdPipelineOrchestratorImpl, appRepositoryImpl, pipelineRepositoryImpl, resourceGroupServiceImpl, enforcerUtilImpl, ciMaterialConfigServiceImpl) pipelineBuilderImpl := pipeline.NewPipelineBuilderImpl(sugaredLogger, materialRepositoryImpl, chartRepositoryImpl, ciPipelineConfigServiceImpl, ciMaterialConfigServiceImpl, appArtifactManagerImpl, devtronAppCMCSServiceImpl, devtronAppStrategyServiceImpl, appDeploymentTypeChangeManagerImpl, cdPipelineConfigServiceImpl, devtronAppConfigServiceImpl) - dbMigrationServiceImpl := pipeline.NewDbMogrationService(sugaredLogger, dbMigrationConfigRepositoryImpl) ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciWorkflowRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, mergeUtil, ciPipelineRepositoryImpl, prePostCiScriptHistoryServiceImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateServiceImpl, appCrudOperationServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, customTagServiceImpl, pluginInputVariableParserImpl, globalPluginServiceImpl) ciLogServiceImpl, err := pipeline.NewCiLogServiceImpl(sugaredLogger, ciServiceImpl, k8sUtil) if err != nil { @@ -534,7 +529,7 @@ func InitializeApp() (*App, error) { imageScanObjectMetaRepositoryImpl := security.NewImageScanObjectMetaRepositoryImpl(db, sugaredLogger) cveStoreRepositoryImpl := security.NewCveStoreRepositoryImpl(db, sugaredLogger) policyServiceImpl := security2.NewPolicyServiceImpl(environmentServiceImpl, sugaredLogger, appRepositoryImpl, pipelineOverrideRepositoryImpl, cvePolicyRepositoryImpl, clusterServiceImplExtended, pipelineRepositoryImpl, imageScanResultRepositoryImpl, imageScanDeployInfoRepositoryImpl, imageScanObjectMetaRepositoryImpl, httpClient, ciArtifactRepositoryImpl, ciCdConfig, imageScanHistoryRepositoryImpl, cveStoreRepositoryImpl, ciTemplateRepositoryImpl) - pipelineConfigRestHandlerImpl := app3.NewPipelineRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, chartServiceImpl, propertiesConfigServiceImpl, dbMigrationServiceImpl, applicationServiceClientImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, environmentServiceImpl, gitRegistryConfigImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, deploymentTemplateServiceImpl, appWorkflowServiceImpl, materialRepositoryImpl, policyServiceImpl, imageScanResultRepositoryImpl, gitProviderRepositoryImpl, argoUserServiceImpl, ciPipelineMaterialRepositoryImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl) + pipelineConfigRestHandlerImpl := app3.NewPipelineRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, chartServiceImpl, propertiesConfigServiceImpl, applicationServiceClientImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, environmentServiceImpl, gitRegistryConfigImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, deploymentTemplateServiceImpl, appWorkflowServiceImpl, materialRepositoryImpl, policyServiceImpl, imageScanResultRepositoryImpl, gitProviderRepositoryImpl, argoUserServiceImpl, ciPipelineMaterialRepositoryImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl) appWorkflowRestHandlerImpl := restHandler.NewAppWorkflowRestHandlerImpl(sugaredLogger, userServiceImpl, appWorkflowServiceImpl, teamServiceImpl, enforcerImpl, pipelineBuilderImpl, appRepositoryImpl, enforcerUtilImpl) webhookEventDataRepositoryImpl := repository.NewWebhookEventDataRepositoryImpl(db) webhookEventDataConfigImpl := pipeline.NewWebhookEventDataConfigImpl(sugaredLogger, webhookEventDataRepositoryImpl) @@ -543,10 +538,6 @@ func InitializeApp() (*App, error) { pipelineHistoryRestHandlerImpl := restHandler.NewPipelineHistoryRestHandlerImpl(sugaredLogger, userServiceImpl, enforcerImpl, pipelineStrategyHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, prePostCiScriptHistoryServiceImpl, prePostCdScriptHistoryServiceImpl, enforcerUtilImpl, deployedConfigurationHistoryServiceImpl) pipelineStatusTimelineRestHandlerImpl := restHandler.NewPipelineStatusTimelineRestHandlerImpl(sugaredLogger, pipelineStatusTimelineServiceImpl, enforcerUtilImpl, enforcerImpl) pipelineConfigRouterImpl := router.NewPipelineRouterImpl(pipelineConfigRestHandlerImpl, appWorkflowRestHandlerImpl, webhookDataRestHandlerImpl, pipelineHistoryRestHandlerImpl, pipelineStatusTimelineRestHandlerImpl) - dbConfigRepositoryImpl := repository.NewDbConfigRepositoryImpl(db, sugaredLogger) - dbConfigServiceImpl := pipeline.NewDbConfigService(dbConfigRepositoryImpl, sugaredLogger) - migrateDbRestHandlerImpl := restHandler.NewMigrateDbRestHandlerImpl(dockerRegistryConfigImpl, sugaredLogger, gitRegistryConfigImpl, dbConfigServiceImpl, userServiceImpl, validate, dbMigrationServiceImpl, enforcerImpl) - migrateDbRouterImpl := router.NewMigrateDbRouterImpl(migrateDbRestHandlerImpl) appStoreVersionValuesRepositoryImpl := appStoreValuesRepository.NewAppStoreVersionValuesRepositoryImpl(sugaredLogger, db) appStoreValuesServiceImpl := service2.NewAppStoreValuesServiceImpl(sugaredLogger, appStoreApplicationVersionRepositoryImpl, installedAppRepositoryImpl, appStoreVersionValuesRepositoryImpl, userServiceImpl) k8sResourceHistoryRepositoryImpl := repository14.NewK8sResourceHistoryRepositoryImpl(db, sugaredLogger) @@ -607,15 +598,8 @@ func InitializeApp() (*App, error) { resourceServiceImpl := ArgoUtil.NewResourceServiceImpl(argoSession) cdRestHandlerImpl := restHandler.NewCDRestHandlerImpl(sugaredLogger, resourceServiceImpl) cdRouterImpl := router.NewCDRouterImpl(sugaredLogger, cdRestHandlerImpl) - jiraAccountRepositoryImpl := repository.NewJiraAccountRepositoryImpl(db) - jiraClientImpl := client4.NewJiraClientImpl(sugaredLogger, httpClient) - accountServiceImpl := jira.NewAccountServiceImpl(sugaredLogger, jiraAccountRepositoryImpl, jiraClientImpl) - accountValidatorImpl := jira.NewAccountValidatorImpl(sugaredLogger, jiraClientImpl) - projectManagementServiceImpl := jira2.NewProjectManagementServiceImpl(sugaredLogger, accountServiceImpl, jiraAccountRepositoryImpl, accountValidatorImpl) - jiraRestHandlerImpl := restHandler.NewJiraRestHandlerImpl(projectManagementServiceImpl, sugaredLogger, userServiceImpl, validate) - projectManagementRouterImpl := router.NewProjectManagementRouterImpl(jiraRestHandlerImpl) deleteServiceFullModeImpl := delete2.NewDeleteServiceFullModeImpl(sugaredLogger, materialRepositoryImpl, gitRegistryConfigImpl, ciTemplateRepositoryImpl, dockerRegistryConfigImpl, dockerArtifactStoreRepositoryImpl) - gitProviderRestHandlerImpl := restHandler.NewGitProviderRestHandlerImpl(dockerRegistryConfigImpl, sugaredLogger, gitRegistryConfigImpl, dbConfigServiceImpl, userServiceImpl, validate, enforcerImpl, teamServiceImpl, deleteServiceFullModeImpl) + gitProviderRestHandlerImpl := restHandler.NewGitProviderRestHandlerImpl(dockerRegistryConfigImpl, sugaredLogger, gitRegistryConfigImpl, userServiceImpl, validate, enforcerImpl, teamServiceImpl, deleteServiceFullModeImpl) gitProviderRouterImpl := router.NewGitProviderRouterImpl(gitProviderRestHandlerImpl) gitHostRepositoryImpl := repository.NewGitHostRepositoryImpl(db) gitHostConfigImpl := pipeline.NewGitHostConfigImpl(gitHostRepositoryImpl, sugaredLogger, attributesServiceImpl) @@ -635,7 +619,7 @@ func InitializeApp() (*App, error) { webhookNotificationServiceImpl := notifier.NewWebhookNotificationServiceImpl(sugaredLogger, webhookNotificationRepositoryImpl, teamServiceImpl, userRepositoryImpl, notificationSettingsRepositoryImpl) sesNotificationServiceImpl := notifier.NewSESNotificationServiceImpl(sugaredLogger, sesNotificationRepositoryImpl, teamServiceImpl, notificationSettingsRepositoryImpl) smtpNotificationServiceImpl := notifier.NewSMTPNotificationServiceImpl(sugaredLogger, smtpNotificationRepositoryImpl, teamServiceImpl, notificationSettingsRepositoryImpl) - notificationRestHandlerImpl := restHandler.NewNotificationRestHandlerImpl(dockerRegistryConfigImpl, sugaredLogger, gitRegistryConfigImpl, dbConfigServiceImpl, userServiceImpl, validate, notificationConfigServiceImpl, slackNotificationServiceImpl, webhookNotificationServiceImpl, sesNotificationServiceImpl, smtpNotificationServiceImpl, enforcerImpl, teamServiceImpl, environmentServiceImpl, pipelineBuilderImpl, enforcerUtilImpl) + notificationRestHandlerImpl := restHandler.NewNotificationRestHandlerImpl(dockerRegistryConfigImpl, sugaredLogger, gitRegistryConfigImpl, userServiceImpl, validate, notificationConfigServiceImpl, slackNotificationServiceImpl, webhookNotificationServiceImpl, sesNotificationServiceImpl, smtpNotificationServiceImpl, enforcerImpl, teamServiceImpl, environmentServiceImpl, pipelineBuilderImpl, enforcerUtilImpl) notificationRouterImpl := router.NewNotificationRouterImpl(notificationRestHandlerImpl) teamRestHandlerImpl := team2.NewTeamRestHandlerImpl(sugaredLogger, teamServiceImpl, userServiceImpl, enforcerImpl, validate, userAuthServiceImpl, deleteServiceExtendedImpl) teamRouterImpl := team2.NewTeamRouterImpl(teamRestHandlerImpl) @@ -688,8 +672,6 @@ func InitializeApp() (*App, error) { chartGroupServiceImpl := service.NewChartGroupServiceImpl(chartGroupEntriesRepositoryImpl, chartGroupReposotoryImpl, sugaredLogger, chartGroupDeploymentRepositoryImpl, installedAppRepositoryImpl, appStoreVersionValuesRepositoryImpl, userAuthServiceImpl) chartGroupRestHandlerImpl := restHandler.NewChartGroupRestHandlerImpl(chartGroupServiceImpl, sugaredLogger, userServiceImpl, enforcerImpl, validate) chartGroupRouterImpl := router.NewChartGroupRouterImpl(chartGroupRestHandlerImpl) - testSuitRestHandlerImpl := restHandler.NewTestSuitRestHandlerImpl(sugaredLogger, userServiceImpl, validate, enforcerImpl, enforcerUtilImpl, eventClientConfig, httpClient) - testSuitRouterImpl := router.NewTestSuitRouterImpl(testSuitRestHandlerImpl) scanToolExecutionHistoryMappingRepositoryImpl := security.NewScanToolExecutionHistoryMappingRepositoryImpl(db, sugaredLogger) imageScanServiceImpl := security2.NewImageScanServiceImpl(sugaredLogger, imageScanHistoryRepositoryImpl, imageScanResultRepositoryImpl, imageScanObjectMetaRepositoryImpl, cveStoreRepositoryImpl, imageScanDeployInfoRepositoryImpl, userServiceImpl, teamRepositoryImpl, appRepositoryImpl, environmentServiceImpl, ciArtifactRepositoryImpl, policyServiceImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, scanToolMetadataRepositoryImpl, scanToolExecutionHistoryMappingRepositoryImpl) imageScanRestHandlerImpl := restHandler.NewImageScanRestHandlerImpl(sugaredLogger, imageScanServiceImpl, userServiceImpl, enforcerImpl, enforcerUtilImpl, environmentServiceImpl) @@ -736,7 +718,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - bulkUpdateRestHandlerImpl := restHandler.NewBulkUpdateRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, bulkUpdateServiceImpl, chartServiceImpl, propertiesConfigServiceImpl, dbMigrationServiceImpl, applicationServiceClientImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, environmentServiceImpl, gitRegistryConfigImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, appWorkflowServiceImpl, materialRepositoryImpl, policyServiceImpl, imageScanResultRepositoryImpl, argoUserServiceImpl) + bulkUpdateRestHandlerImpl := restHandler.NewBulkUpdateRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, bulkUpdateServiceImpl, chartServiceImpl, propertiesConfigServiceImpl, applicationServiceClientImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, environmentServiceImpl, gitRegistryConfigImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, appWorkflowServiceImpl, materialRepositoryImpl, policyServiceImpl, imageScanResultRepositoryImpl, argoUserServiceImpl) bulkUpdateRouterImpl := router.NewBulkUpdateRouterImpl(bulkUpdateRestHandlerImpl) webhookSecretValidatorImpl := git.NewWebhookSecretValidatorImpl(sugaredLogger) webhookEventHandlerImpl := restHandler.NewWebhookEventHandlerImpl(sugaredLogger, gitHostConfigImpl, eventRESTClientImpl, webhookSecretValidatorImpl, webhookEventDataConfigImpl) @@ -820,7 +802,7 @@ func InitializeApp() (*App, error) { return nil, err } ciTriggerCronImpl := cron.NewCiTriggerCronImpl(sugaredLogger, ciTriggerCronConfig, pipelineStageRepositoryImpl, ciHandlerImpl, ciArtifactRepositoryImpl, globalPluginRepositoryImpl) - muxRouter := router.NewMuxRouter(sugaredLogger, pipelineTriggerRouterImpl, pipelineConfigRouterImpl, migrateDbRouterImpl, appListingRouterImpl, environmentRouterImpl, clusterRouterImpl, webhookRouterImpl, userAuthRouterImpl, applicationRouterImpl, cdRouterImpl, projectManagementRouterImpl, gitProviderRouterImpl, gitHostRouterImpl, dockerRegRouterImpl, notificationRouterImpl, teamRouterImpl, gitWebhookHandlerImpl, workflowStatusUpdateHandlerImpl, applicationStatusHandlerImpl, ciEventHandlerImpl, pubSubClientServiceImpl, userRouterImpl, chartRefRouterImpl, configMapRouterImpl, appStoreRouterImpl, chartRepositoryRouterImpl, releaseMetricsRouterImpl, deploymentGroupRouterImpl, batchOperationRouterImpl, chartGroupRouterImpl, testSuitRouterImpl, imageScanRouterImpl, policyRouterImpl, gitOpsConfigRouterImpl, dashboardRouterImpl, attributesRouterImpl, userAttributesRouterImpl, commonRouterImpl, grafanaRouterImpl, ssoLoginRouterImpl, telemetryRouterImpl, telemetryEventClientImplExtended, bulkUpdateRouterImpl, webhookListenerRouterImpl, appRouterImpl, coreAppRouterImpl, helmAppRouterImpl, k8sApplicationRouterImpl, pProfRouterImpl, deploymentConfigRouterImpl, dashboardTelemetryRouterImpl, commonDeploymentRouterImpl, externalLinkRouterImpl, globalPluginRouterImpl, moduleRouterImpl, serverRouterImpl, apiTokenRouterImpl, cdApplicationStatusUpdateHandlerImpl, k8sCapacityRouterImpl, webhookHelmRouterImpl, globalCMCSRouterImpl, userTerminalAccessRouterImpl, jobRouterImpl, ciStatusUpdateCronImpl, resourceGroupingRouterImpl, rbacRoleRouterImpl, scopedVariableRouterImpl, ciTriggerCronImpl) + muxRouter := router.NewMuxRouter(sugaredLogger, pipelineTriggerRouterImpl, pipelineConfigRouterImpl, appListingRouterImpl, environmentRouterImpl, clusterRouterImpl, webhookRouterImpl, userAuthRouterImpl, applicationRouterImpl, cdRouterImpl, gitProviderRouterImpl, gitHostRouterImpl, dockerRegRouterImpl, notificationRouterImpl, teamRouterImpl, gitWebhookHandlerImpl, workflowStatusUpdateHandlerImpl, applicationStatusHandlerImpl, ciEventHandlerImpl, pubSubClientServiceImpl, userRouterImpl, chartRefRouterImpl, configMapRouterImpl, appStoreRouterImpl, chartRepositoryRouterImpl, releaseMetricsRouterImpl, deploymentGroupRouterImpl, batchOperationRouterImpl, chartGroupRouterImpl, imageScanRouterImpl, policyRouterImpl, gitOpsConfigRouterImpl, dashboardRouterImpl, attributesRouterImpl, userAttributesRouterImpl, commonRouterImpl, grafanaRouterImpl, ssoLoginRouterImpl, telemetryRouterImpl, telemetryEventClientImplExtended, bulkUpdateRouterImpl, webhookListenerRouterImpl, appRouterImpl, coreAppRouterImpl, helmAppRouterImpl, k8sApplicationRouterImpl, pProfRouterImpl, deploymentConfigRouterImpl, dashboardTelemetryRouterImpl, commonDeploymentRouterImpl, externalLinkRouterImpl, globalPluginRouterImpl, moduleRouterImpl, serverRouterImpl, apiTokenRouterImpl, cdApplicationStatusUpdateHandlerImpl, k8sCapacityRouterImpl, webhookHelmRouterImpl, globalCMCSRouterImpl, userTerminalAccessRouterImpl, jobRouterImpl, ciStatusUpdateCronImpl, resourceGroupingRouterImpl, rbacRoleRouterImpl, scopedVariableRouterImpl, ciTriggerCronImpl) loggingMiddlewareImpl := util4.NewLoggingMiddlewareImpl(userServiceImpl) mainApp := NewApp(muxRouter, sugaredLogger, sseSSE, syncedEnforcer, db, pubSubClientServiceImpl, sessionManager, posthogClient, loggingMiddlewareImpl) return mainApp, nil From 28b07fd7acfcc20184a45e0ca818a0bb24c73dac Mon Sep 17 00:00:00 2001 From: Nishant <58689354+nishant-d@users.noreply.github.com> Date: Mon, 15 Jan 2024 17:01:16 +0530 Subject: [PATCH 05/64] chore: App store dead code cleanup and restructuring (#4497) * moved chart-group in seperate code * removed unused dependency * removed dead code * extracted resource tree * moved notes * resource movement * removed unused code * removed unused dependency * commit methods * extracted status update * chore: clean up unused dead code * updated: EA mode docker file * updated: migration number --------- Co-authored-by: Ash-exp --- DockerfileEA | 2 +- Wire.go | 9 +- api/appStore/InstalledAppRestHandler.go | 3 +- api/restHandler/ChartGroupRestHandler.go | 16 +- cmd/external-app/wire.go | 5 +- cmd/external-app/wire_gen.go | 7 +- pkg/appStore/bean/bean.go | 21 - .../ChartGroupService.go | 29 +- pkg/appStore/chartGroup/bean.go | 23 + .../repository/ChartGroup.go | 0 .../repository/ChartGroupDeployment.go | 0 .../repository/ChartGroupEntry.go | 0 .../AppStoreDeploymentFullModeService.go | 284 ++-------- .../service/AppStoreDeploymentService.go | 13 +- .../service/AppStoreDeploymentService_test.go | 3 +- .../deployment/service/InstalledAppService.go | 492 +----------------- .../service/InstalledAppService_test.go | 3 +- pkg/appStore/deployment/service/Notes.go | 118 +++++ .../deployment/service/ResourceTree.go | 334 ++++++++++++ .../AppStoreDeploymentArgoCdService.go | 295 +---------- .../tool/AppStoreDeploymentHelmService.go | 113 +--- .../tool/DeploymentStatusService.go | 122 +++++ ...n.up.sql => 211_unused_integration.up.sql} | 0 ...n.sql => 211_unused_intergration.down.sql} | 0 wire_gen.go | 39 +- 25 files changed, 732 insertions(+), 1199 deletions(-) rename pkg/appStore/{deployment/service => chartGroup}/ChartGroupService.go (95%) create mode 100644 pkg/appStore/chartGroup/bean.go rename pkg/appStore/{deployment => chartGroup}/repository/ChartGroup.go (100%) rename pkg/appStore/{deployment => chartGroup}/repository/ChartGroupDeployment.go (100%) rename pkg/appStore/{deployment => chartGroup}/repository/ChartGroupEntry.go (100%) create mode 100644 pkg/appStore/deployment/service/Notes.go create mode 100644 pkg/appStore/deployment/service/ResourceTree.go rename pkg/appStore/deployment/tool/{gitops => }/AppStoreDeploymentArgoCdService.go (66%) create mode 100644 pkg/appStore/deployment/tool/DeploymentStatusService.go rename scripts/sql/{210_unused_integration.up.sql => 211_unused_integration.up.sql} (100%) rename scripts/sql/{210_unused_intergration.down.sql => 211_unused_intergration.down.sql} (100%) diff --git a/DockerfileEA b/DockerfileEA index 0fbc7b85ec..74827c1e35 100644 --- a/DockerfileEA +++ b/DockerfileEA @@ -21,7 +21,7 @@ COPY --from=build-env /go/src/github.com/devtron-labs/devtron/vendor/github.com COPY --from=build-env /go/src/github.com/devtron-labs/devtron/scripts/devtron-reference-helm-charts scripts/devtron-reference-helm-charts COPY --from=build-env /go/src/github.com/devtron-labs/devtron/scripts/sql scripts/sql COPY --from=build-env /go/src/github.com/devtron-labs/devtron/scripts/casbin scripts/casbin -COPY --from=build-env /go/src/github.com/devtron-labs/devtron/scripts/argo-assets/APPLICATION_TEMPLATE.JSON scripts/argo-assets/APPLICATION_TEMPLATE.JSON +COPY --from=build-env /go/src/github.com/devtron-labs/devtron/scripts/argo-assets/APPLICATION_TEMPLATE.tmpl scripts/argo-assets/APPLICATION_TEMPLATE.tmpl RUN useradd -ms /bin/bash devtron RUN chown -R devtron:devtron ./devtron-ea diff --git a/Wire.go b/Wire.go index 69890f5668..caeb9157b9 100644 --- a/Wire.go +++ b/Wire.go @@ -85,10 +85,11 @@ import ( "github.com/devtron-labs/devtron/pkg/appClone/batch" "github.com/devtron-labs/devtron/pkg/appStatus" appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" + "github.com/devtron-labs/devtron/pkg/appStore/chartGroup" + repository4 "github.com/devtron-labs/devtron/pkg/appStore/chartGroup/repository" appStoreDeploymentFullMode "github.com/devtron-labs/devtron/pkg/appStore/deployment/fullMode" - repository4 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" "github.com/devtron-labs/devtron/pkg/appStore/deployment/service" - appStoreDeploymentGitopsTool "github.com/devtron-labs/devtron/pkg/appStore/deployment/tool/gitops" + appStoreDeploymentGitopsTool "github.com/devtron-labs/devtron/pkg/appStore/deployment/tool" "github.com/devtron-labs/devtron/pkg/appWorkflow" "github.com/devtron-labs/devtron/pkg/attributes" "github.com/devtron-labs/devtron/pkg/bulkAction" @@ -588,8 +589,8 @@ func InitializeApp() (*App, error) { wire.Bind(new(repository4.ChartGroupReposotory), new(*repository4.ChartGroupReposotoryImpl)), repository4.NewChartGroupEntriesRepositoryImpl, wire.Bind(new(repository4.ChartGroupEntriesRepository), new(*repository4.ChartGroupEntriesRepositoryImpl)), - service.NewChartGroupServiceImpl, - wire.Bind(new(service.ChartGroupService), new(*service.ChartGroupServiceImpl)), + chartGroup.NewChartGroupServiceImpl, + wire.Bind(new(chartGroup.ChartGroupService), new(*chartGroup.ChartGroupServiceImpl)), restHandler.NewChartGroupRestHandlerImpl, wire.Bind(new(restHandler.ChartGroupRestHandler), new(*restHandler.ChartGroupRestHandlerImpl)), router.NewChartGroupRouterImpl, diff --git a/api/appStore/InstalledAppRestHandler.go b/api/appStore/InstalledAppRestHandler.go index b44f3e9056..c6e1427c8d 100644 --- a/api/appStore/InstalledAppRestHandler.go +++ b/api/appStore/InstalledAppRestHandler.go @@ -38,6 +38,7 @@ import ( util2 "github.com/devtron-labs/devtron/internal/util" app2 "github.com/devtron-labs/devtron/pkg/app" appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" + "github.com/devtron-labs/devtron/pkg/appStore/chartGroup" "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" "github.com/devtron-labs/devtron/pkg/appStore/deployment/service" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" @@ -325,7 +326,7 @@ func (handler *InstalledAppRestHandlerImpl) DeployBulk(w http.ResponseWriter, r return } decoder := json.NewDecoder(r.Body) - var request appStoreBean.ChartGroupInstallRequest + var request chartGroup.ChartGroupInstallRequest err = decoder.Decode(&request) if err != nil { handler.Logger.Errorw("request err, DeployBulk", "err", err, "payload", request) diff --git a/api/restHandler/ChartGroupRestHandler.go b/api/restHandler/ChartGroupRestHandler.go index 9a0b2dc03e..0e496bea65 100644 --- a/api/restHandler/ChartGroupRestHandler.go +++ b/api/restHandler/ChartGroupRestHandler.go @@ -24,7 +24,7 @@ import ( "strconv" "github.com/devtron-labs/devtron/api/restHandler/common" - "github.com/devtron-labs/devtron/pkg/appStore/deployment/service" + "github.com/devtron-labs/devtron/pkg/appStore/chartGroup" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" "github.com/devtron-labs/devtron/pkg/auth/user" "github.com/gorilla/mux" @@ -35,14 +35,14 @@ import ( const CHART_GROUP_DELETE_SUCCESS_RESP = "Chart group deleted successfully." type ChartGroupRestHandlerImpl struct { - ChartGroupService service.ChartGroupService + ChartGroupService chartGroup.ChartGroupService Logger *zap.SugaredLogger userAuthService user.UserService enforcer casbin.Enforcer validator *validator.Validate } -func NewChartGroupRestHandlerImpl(ChartGroupService service.ChartGroupService, +func NewChartGroupRestHandlerImpl(ChartGroupService chartGroup.ChartGroupService, Logger *zap.SugaredLogger, userAuthService user.UserService, enforcer casbin.Enforcer, validator *validator.Validate) *ChartGroupRestHandlerImpl { return &ChartGroupRestHandlerImpl{ @@ -72,7 +72,7 @@ func (impl *ChartGroupRestHandlerImpl) CreateChartGroup(w http.ResponseWriter, r return } decoder := json.NewDecoder(r.Body) - var request service.ChartGroupBean + var request chartGroup.ChartGroupBean err = decoder.Decode(&request) if err != nil { impl.Logger.Errorw("request err, CreateChartGroup", "err", err, "payload", request) @@ -101,7 +101,7 @@ func (impl *ChartGroupRestHandlerImpl) CreateChartGroup(w http.ResponseWriter, r if err != nil { impl.Logger.Errorw("service err, CreateChartGroup", "err", err, "payload", request) statusCode := http.StatusInternalServerError - if service.AppNameAlreadyExistsError == err.Error() { + if chartGroup.AppNameAlreadyExistsError == err.Error() { statusCode = http.StatusBadRequest } common.WriteJsonResp(w, err, nil, statusCode) @@ -117,7 +117,7 @@ func (impl *ChartGroupRestHandlerImpl) UpdateChartGroup(w http.ResponseWriter, r return } decoder := json.NewDecoder(r.Body) - var request service.ChartGroupBean + var request chartGroup.ChartGroupBean err = decoder.Decode(&request) if err != nil { impl.Logger.Errorw("request err, UpdateChartGroup", "err", err, "payload", request) @@ -158,7 +158,7 @@ func (impl *ChartGroupRestHandlerImpl) SaveChartGroupEntries(w http.ResponseWrit return } decoder := json.NewDecoder(r.Body) - var request service.ChartGroupBean + var request chartGroup.ChartGroupBean err = decoder.Decode(&request) if err != nil { impl.Logger.Errorw("request err, SaveChartGroupEntries", "err", err, "payload", request) @@ -326,7 +326,7 @@ func (impl *ChartGroupRestHandlerImpl) DeleteChartGroup(w http.ResponseWriter, r return } decoder := json.NewDecoder(r.Body) - var request service.ChartGroupBean + var request chartGroup.ChartGroupBean err = decoder.Decode(&request) if err != nil { impl.Logger.Errorw("request err, DeleteChartGroup", "err", err, "payload", request) diff --git a/cmd/external-app/wire.go b/cmd/external-app/wire.go index 044f9c0d98..e03344eaff 100644 --- a/cmd/external-app/wire.go +++ b/cmd/external-app/wire.go @@ -39,9 +39,8 @@ import ( "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app" appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" - repository3 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" + repository3 "github.com/devtron-labs/devtron/pkg/appStore/chartGroup/repository" appStoreDeploymentTool "github.com/devtron-labs/devtron/pkg/appStore/deployment/tool" - appStoreDeploymentGitopsTool "github.com/devtron-labs/devtron/pkg/appStore/deployment/tool/gitops" "github.com/devtron-labs/devtron/pkg/attributes" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" delete2 "github.com/devtron-labs/devtron/pkg/delete" @@ -138,7 +137,7 @@ func InitializeApp() (*App, error) { // // needed for enforcer util ends // binding gitops to helm (for hyperion) - wire.Bind(new(appStoreDeploymentGitopsTool.AppStoreDeploymentArgoCdService), new(*appStoreDeploymentTool.AppStoreDeploymentHelmServiceImpl)), + wire.Bind(new(appStoreDeploymentTool.AppStoreDeploymentArgoCdService), new(*appStoreDeploymentTool.AppStoreDeploymentHelmServiceImpl)), wire.Value(chartRepoRepository.RefChartDir("scripts/devtron-reference-helm-charts")), diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index c95ee93e35..9e3eb8ea36 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -46,6 +46,7 @@ import ( "github.com/devtron-labs/devtron/pkg/apiToken" app2 "github.com/devtron-labs/devtron/pkg/app" "github.com/devtron-labs/devtron/pkg/appStore/bean" + repository8 "github.com/devtron-labs/devtron/pkg/appStore/chartGroup/repository" "github.com/devtron-labs/devtron/pkg/appStore/chartProvider" "github.com/devtron-labs/devtron/pkg/appStore/deployment/common" repository4 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" @@ -274,9 +275,9 @@ func InitializeApp() (*App, error) { appStoreValuesServiceImpl := service2.NewAppStoreValuesServiceImpl(sugaredLogger, appStoreApplicationVersionRepositoryImpl, installedAppRepositoryImpl, appStoreVersionValuesRepositoryImpl, userServiceImpl) appStoreValuesRestHandlerImpl := appStoreValues.NewAppStoreValuesRestHandlerImpl(sugaredLogger, userServiceImpl, appStoreValuesServiceImpl) appStoreValuesRouterImpl := appStoreValues.NewAppStoreValuesRouterImpl(appStoreValuesRestHandlerImpl) - chartGroupDeploymentRepositoryImpl := repository4.NewChartGroupDeploymentRepositoryImpl(db, sugaredLogger) + chartGroupDeploymentRepositoryImpl := repository8.NewChartGroupDeploymentRepositoryImpl(db, sugaredLogger) clusterInstalledAppsRepositoryImpl := repository4.NewClusterInstalledAppsRepositoryImpl(db, sugaredLogger) - appStoreDeploymentHelmServiceImpl := appStoreDeploymentTool.NewAppStoreDeploymentHelmServiceImpl(sugaredLogger, helmAppServiceImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, helmAppClientImpl, installedAppRepositoryImpl, appStoreDeploymentCommonServiceImpl, ociRegistryConfigRepositoryImpl) + appStoreDeploymentHelmServiceImpl := appStoreDeploymentTool.NewAppStoreDeploymentHelmServiceImpl(sugaredLogger, helmAppServiceImpl, appStoreApplicationVersionRepositoryImpl, helmAppClientImpl, installedAppRepositoryImpl, appStoreDeploymentCommonServiceImpl, ociRegistryConfigRepositoryImpl) installedAppVersionHistoryRepositoryImpl := repository4.NewInstalledAppVersionHistoryRepositoryImpl(sugaredLogger, db) deploymentServiceTypeConfig, err := service3.GetDeploymentServiceTypeConfig() if err != nil { @@ -286,7 +287,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - appStoreDeploymentServiceImpl := service3.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, clusterInstalledAppsRepositoryImpl, appRepositoryImpl, appStoreDeploymentHelmServiceImpl, appStoreDeploymentHelmServiceImpl, environmentServiceImpl, clusterServiceImpl, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, globalEnvVariables, installedAppVersionHistoryRepositoryImpl, gitOpsConfigRepositoryImpl, attributesServiceImpl, deploymentServiceTypeConfig, chartTemplateServiceImpl, acdConfig) + appStoreDeploymentServiceImpl := service3.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, clusterInstalledAppsRepositoryImpl, appRepositoryImpl, appStoreDeploymentHelmServiceImpl, appStoreDeploymentHelmServiceImpl, environmentServiceImpl, clusterServiceImpl, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, installedAppVersionHistoryRepositoryImpl, gitOpsConfigRepositoryImpl, attributesServiceImpl, deploymentServiceTypeConfig, chartTemplateServiceImpl, acdConfig) appStoreDeploymentRestHandlerImpl := appStoreDeployment.NewAppStoreDeploymentRestHandlerImpl(sugaredLogger, userServiceImpl, enforcerImpl, enforcerUtilImpl, enforcerUtilHelmImpl, appStoreDeploymentServiceImpl, validate, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, helmUserServiceImpl, attributesServiceImpl) appStoreDeploymentRouterImpl := appStoreDeployment.NewAppStoreDeploymentRouterImpl(appStoreDeploymentRestHandlerImpl) chartProviderServiceImpl := chartProvider.NewChartProviderServiceImpl(sugaredLogger, chartRepoRepositoryImpl, chartRepositoryServiceImpl, dockerArtifactStoreRepositoryImpl, ociRegistryConfigRepositoryImpl) diff --git a/pkg/appStore/bean/bean.go b/pkg/appStore/bean/bean.go index 19dde14013..373cc85deb 100644 --- a/pkg/appStore/bean/bean.go +++ b/pkg/appStore/bean/bean.go @@ -126,27 +126,6 @@ type InstallAppVersionChartRepoDTO struct { Password string `json:"-"` } -// / bean for v2 -type ChartGroupInstallRequest struct { - ProjectId int `json:"projectId" validate:"required,number"` - ChartGroupInstallChartRequest []*ChartGroupInstallChartRequest `json:"charts" validate:"dive,required"` - ChartGroupId int `json:"chartGroupId"` //optional - UserId int32 `json:"-"` -} - -type ChartGroupInstallChartRequest struct { - AppName string `json:"appName,omitempty" validate:"name-component,max=100" ` - EnvironmentId int `json:"environmentId,omitempty" validate:"required,number" ` - AppStoreVersion int `json:"appStoreVersion,omitempty,notnull" validate:"required,number" ` - ValuesOverrideYaml string `json:"valuesOverrideYaml,omitempty"` //optional - ReferenceValueId int `json:"referenceValueId, omitempty" validate:"required,number"` - ReferenceValueKind string `json:"referenceValueKind, omitempty" validate:"oneof=DEFAULT TEMPLATE DEPLOYED"` - ChartGroupEntryId int `json:"chartGroupEntryId"` //optional - DefaultClusterComponent bool `json:"-"` -} -type ChartGroupInstallAppRes struct { -} - // / type RefChartProxyDir string diff --git a/pkg/appStore/deployment/service/ChartGroupService.go b/pkg/appStore/chartGroup/ChartGroupService.go similarity index 95% rename from pkg/appStore/deployment/service/ChartGroupService.go rename to pkg/appStore/chartGroup/ChartGroupService.go index 02e396c19a..7036d946a5 100644 --- a/pkg/appStore/deployment/service/ChartGroupService.go +++ b/pkg/appStore/chartGroup/ChartGroupService.go @@ -15,13 +15,14 @@ * */ -package service +package chartGroup import ( "errors" "time" appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" + repository2 "github.com/devtron-labs/devtron/pkg/appStore/chartGroup/repository" "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" appStoreValuesRepository "github.com/devtron-labs/devtron/pkg/appStore/values/repository" "github.com/devtron-labs/devtron/pkg/auth/user" @@ -32,18 +33,18 @@ import ( ) type ChartGroupServiceImpl struct { - chartGroupEntriesRepository repository.ChartGroupEntriesRepository - chartGroupRepository repository.ChartGroupReposotory + chartGroupEntriesRepository repository2.ChartGroupEntriesRepository + chartGroupRepository repository2.ChartGroupReposotory Logger *zap.SugaredLogger - chartGroupDeploymentRepository repository.ChartGroupDeploymentRepository + chartGroupDeploymentRepository repository2.ChartGroupDeploymentRepository installedAppRepository repository.InstalledAppRepository appStoreVersionValuesRepository appStoreValuesRepository.AppStoreVersionValuesRepository userAuthService user.UserAuthService } -func NewChartGroupServiceImpl(chartGroupEntriesRepository repository.ChartGroupEntriesRepository, - chartGroupRepository repository.ChartGroupReposotory, - Logger *zap.SugaredLogger, chartGroupDeploymentRepository repository.ChartGroupDeploymentRepository, +func NewChartGroupServiceImpl(chartGroupEntriesRepository repository2.ChartGroupEntriesRepository, + chartGroupRepository repository2.ChartGroupReposotory, + Logger *zap.SugaredLogger, chartGroupDeploymentRepository repository2.ChartGroupDeploymentRepository, installedAppRepository repository.InstalledAppRepository, appStoreVersionValuesRepository appStoreValuesRepository.AppStoreVersionValuesRepository, userAuthService user.UserAuthService) *ChartGroupServiceImpl { return &ChartGroupServiceImpl{ chartGroupEntriesRepository: chartGroupEntriesRepository, @@ -125,7 +126,7 @@ func (impl *ChartGroupServiceImpl) CreateChartGroup(req *ChartGroupBean) (*Chart return nil, errors.New(AppNameAlreadyExistsError) } - chartGrouModel := &repository.ChartGroup{ + chartGrouModel := &repository2.ChartGroup{ Name: req.Name, Description: req.Description, AuditLog: sql.AuditLog{ @@ -146,7 +147,7 @@ func (impl *ChartGroupServiceImpl) CreateChartGroup(req *ChartGroupBean) (*Chart func (impl *ChartGroupServiceImpl) UpdateChartGroup(req *ChartGroupBean) (*ChartGroupBean, error) { impl.Logger.Debugw("chart group update request", "req", req) - chartGrouModel := &repository.ChartGroup{ + chartGrouModel := &repository2.ChartGroup{ Name: req.Name, Description: req.Description, Id: req.Id, @@ -182,7 +183,7 @@ func (impl *ChartGroupServiceImpl) SaveChartGroupEntries(req *ChartGroupBean) (* newEntries = append(newEntries, entryBean) } } - var updateEntries []*repository.ChartGroupEntry + var updateEntries []*repository2.ChartGroupEntry for _, existingEntry := range group.ChartGroupEntries { if entry, ok := oldEntriesMap[existingEntry.Id]; ok { //update @@ -197,9 +198,9 @@ func (impl *ChartGroupServiceImpl) SaveChartGroupEntries(req *ChartGroupBean) (* updateEntries = append(updateEntries, existingEntry) } - var createEntries []*repository.ChartGroupEntry + var createEntries []*repository2.ChartGroupEntry for _, entryBean := range newEntries { - entry := &repository.ChartGroupEntry{ + entry := &repository2.ChartGroupEntry{ AppStoreValuesVersionId: entryBean.AppStoreValuesVersionId, AppStoreApplicationVersionId: entryBean.AppStoreApplicationVersionId, ChartGroupId: group.Id, @@ -243,7 +244,7 @@ func (impl *ChartGroupServiceImpl) GetChartGroupWithChartMetaData(chartGroupId i return chartGroupRes, err } -func (impl *ChartGroupServiceImpl) charterEntryAdopter(chartGroupEntry *repository.ChartGroupEntry) *ChartGroupEntryBean { +func (impl *ChartGroupServiceImpl) charterEntryAdopter(chartGroupEntry *repository2.ChartGroupEntry) *ChartGroupEntryBean { var referenceType string var valueVersionName string @@ -342,7 +343,7 @@ func (impl *ChartGroupServiceImpl) GetChartGroupWithInstallationDetail(chartGrou impl.Logger.Errorw("error in finding deployment", "chartGroupId", chartGroupId, "err", err) return nil, err } - groupDeploymentMap := make(map[string][]*repository.ChartGroupDeployment) + groupDeploymentMap := make(map[string][]*repository2.ChartGroupDeployment) for _, deployment := range deployments { groupDeploymentMap[deployment.GroupInstallationId] = append(groupDeploymentMap[deployment.GroupInstallationId], deployment) } diff --git a/pkg/appStore/chartGroup/bean.go b/pkg/appStore/chartGroup/bean.go new file mode 100644 index 0000000000..c385e358da --- /dev/null +++ b/pkg/appStore/chartGroup/bean.go @@ -0,0 +1,23 @@ +package chartGroup + +// / bean for v2 +type ChartGroupInstallRequest struct { + ProjectId int `json:"projectId" validate:"required,number"` + ChartGroupInstallChartRequest []*ChartGroupInstallChartRequest `json:"charts" validate:"dive,required"` + ChartGroupId int `json:"chartGroupId"` //optional + UserId int32 `json:"-"` +} + +type ChartGroupInstallChartRequest struct { + AppName string `json:"appName,omitempty" validate:"name-component,max=100" ` + EnvironmentId int `json:"environmentId,omitempty" validate:"required,number" ` + AppStoreVersion int `json:"appStoreVersion,omitempty,notnull" validate:"required,number" ` + ValuesOverrideYaml string `json:"valuesOverrideYaml,omitempty"` //optional + ReferenceValueId int `json:"referenceValueId, omitempty" validate:"required,number"` + ReferenceValueKind string `json:"referenceValueKind, omitempty" validate:"oneof=DEFAULT TEMPLATE DEPLOYED"` + ChartGroupEntryId int `json:"chartGroupEntryId"` //optional + DefaultClusterComponent bool `json:"-"` +} + +type ChartGroupInstallAppRes struct { +} diff --git a/pkg/appStore/deployment/repository/ChartGroup.go b/pkg/appStore/chartGroup/repository/ChartGroup.go similarity index 100% rename from pkg/appStore/deployment/repository/ChartGroup.go rename to pkg/appStore/chartGroup/repository/ChartGroup.go diff --git a/pkg/appStore/deployment/repository/ChartGroupDeployment.go b/pkg/appStore/chartGroup/repository/ChartGroupDeployment.go similarity index 100% rename from pkg/appStore/deployment/repository/ChartGroupDeployment.go rename to pkg/appStore/chartGroup/repository/ChartGroupDeployment.go diff --git a/pkg/appStore/deployment/repository/ChartGroupEntry.go b/pkg/appStore/chartGroup/repository/ChartGroupEntry.go similarity index 100% rename from pkg/appStore/deployment/repository/ChartGroupEntry.go rename to pkg/appStore/chartGroup/repository/ChartGroupEntry.go diff --git a/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go b/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go index 0899e6d7d0..fd28cedde9 100644 --- a/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go +++ b/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go @@ -24,13 +24,9 @@ import ( "fmt" "github.com/devtron-labs/common-lib/pubsub-lib" "github.com/devtron-labs/common-lib/pubsub-lib/model" - "path" - "regexp" "time" - "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/client/argocdServer" - repository3 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/pkg/app/status" appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" @@ -40,7 +36,6 @@ import ( repository5 "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/sql" util2 "github.com/devtron-labs/devtron/pkg/util" - util3 "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/argo" "github.com/go-pg/pg" @@ -51,9 +46,6 @@ import ( "github.com/devtron-labs/devtron/client/argocdServer/repository" "github.com/devtron-labs/devtron/internal/util" "go.uber.org/zap" - "k8s.io/helm/pkg/chartutil" - "k8s.io/helm/pkg/proto/hapi/chart" - "sigs.k8s.io/yaml" ) const ( @@ -61,52 +53,39 @@ const ( CLUSTER_COMPONENT_DIR_PATH = "/cluster/component" ) +// ACD operation and git operation type AppStoreDeploymentFullModeService interface { - AppStoreDeployOperationGIT(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, *util.ChartGitAttribute, error) AppStoreDeployOperationACD(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, chartGitAttr *util.ChartGitAttribute, ctx context.Context, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) + //TRACE usages and delete if unused RegisterInArgo(chartGitAttribute *util.ChartGitAttribute, ctx context.Context) error - SyncACD(acdAppName string, ctx context.Context) UpdateValuesYaml(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) UpdateRequirementYaml(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, appStoreAppVersion *appStoreDiscoverRepository.AppStoreApplicationVersion) error GetGitOpsRepoName(appName string, environmentName string) (string, error) - SubscribeHelmInstallStatus() error } type AppStoreDeploymentFullModeServiceImpl struct { - logger *zap.SugaredLogger - chartTemplateService util.ChartTemplateService - refChartDir appStoreBean.RefChartProxyDir - repositoryService repository.ServiceClient - appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository - environmentRepository repository5.EnvironmentRepository - acdClient application2.ServiceClient - ArgoK8sClient argocdServer.ArgoK8sClient - gitFactory *util.GitFactory - aCDAuthConfig *util2.ACDAuthConfig - globalEnvVariables *util3.GlobalEnvVariables - installedAppRepository repository4.InstalledAppRepository - tokenCache *util2.TokenCache - argoUserService argo.ArgoUserService - gitOpsConfigRepository repository3.GitOpsConfigRepository - pipelineStatusTimelineService status.PipelineStatusTimelineService - appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService - argoClientWrapperService argocdServer.ArgoClientWrapperService - pubSubClient *pubsub_lib.PubSubClientServiceImpl - installedAppRepositoryHistory repository4.InstalledAppVersionHistoryRepository - ACDConfig *argocdServer.ACDConfig + logger *zap.SugaredLogger + chartTemplateService util.ChartTemplateService + repositoryService repository.ServiceClient + acdClient application2.ServiceClient + ArgoK8sClient argocdServer.ArgoK8sClient + aCDAuthConfig *util2.ACDAuthConfig + argoUserService argo.ArgoUserService + pipelineStatusTimelineService status.PipelineStatusTimelineService + appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService + argoClientWrapperService argocdServer.ArgoClientWrapperService + pubSubClient *pubsub_lib.PubSubClientServiceImpl + installedAppRepositoryHistory repository4.InstalledAppVersionHistoryRepository + ACDConfig *argocdServer.ACDConfig } func NewAppStoreDeploymentFullModeServiceImpl(logger *zap.SugaredLogger, - chartTemplateService util.ChartTemplateService, refChartDir appStoreBean.RefChartProxyDir, + chartTemplateService util.ChartTemplateService, repositoryService repository.ServiceClient, - appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository, - environmentRepository repository5.EnvironmentRepository, acdClient application2.ServiceClient, argoK8sClient argocdServer.ArgoK8sClient, - gitFactory *util.GitFactory, aCDAuthConfig *util2.ACDAuthConfig, - globalEnvVariables *util3.GlobalEnvVariables, - installedAppRepository repository4.InstalledAppRepository, tokenCache *util2.TokenCache, - argoUserService argo.ArgoUserService, gitOpsConfigRepository repository3.GitOpsConfigRepository, + aCDAuthConfig *util2.ACDAuthConfig, + argoUserService argo.ArgoUserService, pipelineStatusTimelineService status.PipelineStatusTimelineService, appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService, argoClientWrapperService argocdServer.ArgoClientWrapperService, @@ -115,210 +94,27 @@ func NewAppStoreDeploymentFullModeServiceImpl(logger *zap.SugaredLogger, ACDConfig *argocdServer.ACDConfig, ) *AppStoreDeploymentFullModeServiceImpl { appStoreDeploymentFullModeServiceImpl := &AppStoreDeploymentFullModeServiceImpl{ - logger: logger, - chartTemplateService: chartTemplateService, - refChartDir: refChartDir, - repositoryService: repositoryService, - appStoreApplicationVersionRepository: appStoreApplicationVersionRepository, - environmentRepository: environmentRepository, - acdClient: acdClient, - ArgoK8sClient: argoK8sClient, - gitFactory: gitFactory, - aCDAuthConfig: aCDAuthConfig, - globalEnvVariables: globalEnvVariables, - installedAppRepository: installedAppRepository, - tokenCache: tokenCache, - argoUserService: argoUserService, - gitOpsConfigRepository: gitOpsConfigRepository, - pipelineStatusTimelineService: pipelineStatusTimelineService, - appStoreDeploymentCommonService: appStoreDeploymentCommonService, - argoClientWrapperService: argoClientWrapperService, - pubSubClient: pubSubClient, - installedAppRepositoryHistory: installedAppRepositoryHistory, - ACDConfig: ACDConfig, - } - err := appStoreDeploymentFullModeServiceImpl.SubscribeHelmInstallStatus() + logger: logger, + chartTemplateService: chartTemplateService, + repositoryService: repositoryService, + acdClient: acdClient, + ArgoK8sClient: argoK8sClient, + aCDAuthConfig: aCDAuthConfig, + argoUserService: argoUserService, + pipelineStatusTimelineService: pipelineStatusTimelineService, + appStoreDeploymentCommonService: appStoreDeploymentCommonService, + argoClientWrapperService: argoClientWrapperService, + pubSubClient: pubSubClient, + installedAppRepositoryHistory: installedAppRepositoryHistory, + ACDConfig: ACDConfig, + } + err := appStoreDeploymentFullModeServiceImpl.subscribeHelmInstallStatus() if err != nil { return nil } return appStoreDeploymentFullModeServiceImpl } -func (impl AppStoreDeploymentFullModeServiceImpl) AppStoreDeployOperationGIT(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, *util.ChartGitAttribute, error) { - appStoreAppVersion, err := impl.appStoreApplicationVersionRepository.FindById(installAppVersionRequest.AppStoreVersion) - if err != nil { - impl.logger.Errorw("fetching error", "err", err) - return installAppVersionRequest, nil, err - } - - environment, err := impl.environmentRepository.FindById(installAppVersionRequest.EnvironmentId) - if err != nil { - impl.logger.Errorw("fetching error", "err", err) - return installAppVersionRequest, nil, err - } - - //STEP 1: Commit and PUSH on Gitlab - template := appStoreBean.CHART_PROXY_TEMPLATE - chartPath := path.Join(string(impl.refChartDir), template) - valid, err := chartutil.IsChartDir(chartPath) - if err != nil || !valid { - impl.logger.Errorw("invalid base chart", "dir", chartPath, "err", err) - return installAppVersionRequest, nil, err - } - chartMeta := &chart.Metadata{ - Name: installAppVersionRequest.AppName, - Version: "1.0.1", - } - _, chartGitAttr, err := impl.chartTemplateService.CreateChartProxy(chartMeta, chartPath, environment.Name, installAppVersionRequest) - if err != nil { - return installAppVersionRequest, nil, err - } - - //STEP 3 - update requirements and values - argocdAppName := installAppVersionRequest.AppName + "-" + environment.Name - dependency := appStoreBean.Dependency{ - Name: appStoreAppVersion.AppStore.Name, - Version: appStoreAppVersion.Version, - } - if appStoreAppVersion.AppStore.ChartRepo != nil { - dependency.Repository = appStoreAppVersion.AppStore.ChartRepo.Url - } - var dependencies []appStoreBean.Dependency - dependencies = append(dependencies, dependency) - requirementDependencies := &appStoreBean.Dependencies{ - Dependencies: dependencies, - } - requirementDependenciesByte, err := json.Marshal(requirementDependencies) - if err != nil { - return installAppVersionRequest, nil, err - } - requirementDependenciesByte, err = yaml.JSONToYAML(requirementDependenciesByte) - if err != nil { - return installAppVersionRequest, nil, err - } - - gitOpsRepoName := impl.chartTemplateService.GetGitOpsRepoName(installAppVersionRequest.AppName) - //getting username & emailId for commit author data - userEmailId, userName := impl.chartTemplateService.GetUserEmailIdAndNameForGitOpsCommit(installAppVersionRequest.UserId) - gitOpsConfigBitbucket, err := impl.gitOpsConfigRepository.GetGitOpsConfigByProvider(util.BITBUCKET_PROVIDER) - if err != nil { - if err == pg.ErrNoRows { - gitOpsConfigBitbucket.BitBucketWorkspaceId = "" - } else { - return installAppVersionRequest, nil, err - } - } - requirmentYamlConfig := &util.ChartConfig{ - FileName: appStoreBean.REQUIREMENTS_YAML_FILE, - FileContent: string(requirementDependenciesByte), - ChartName: chartMeta.Name, - ChartLocation: argocdAppName, - ChartRepoName: gitOpsRepoName, - ReleaseMessage: fmt.Sprintf("release-%d-env-%d ", appStoreAppVersion.Id, environment.Id), - UserEmailId: userEmailId, - UserName: userName, - } - gitOpsConfig := &bean.GitOpsConfigDto{BitBucketWorkspaceId: gitOpsConfigBitbucket.BitBucketWorkspaceId} - _, _, err = impl.gitFactory.Client.CommitValues(requirmentYamlConfig, gitOpsConfig) - if err != nil { - impl.logger.Errorw("error in git commit", "err", err) - return installAppVersionRequest, nil, err - } - - //GIT PULL - space := regexp.MustCompile(`\s+`) - appStoreName := space.ReplaceAllString(chartMeta.Name, "-") - clonedDir := impl.gitFactory.GitWorkingDir + "" + appStoreName - err = impl.chartTemplateService.GitPull(clonedDir, chartGitAttr.RepoUrl, appStoreName) - if err != nil { - impl.logger.Errorw("error in git pull", "err", err) - return installAppVersionRequest, nil, err - } - - //update values yaml in chart - ValuesOverrideByte, err := yaml.YAMLToJSON([]byte(installAppVersionRequest.ValuesOverrideYaml)) - if err != nil { - impl.logger.Errorw("error in json patch", "err", err) - return installAppVersionRequest, nil, err - } - - var dat map[string]interface{} - err = json.Unmarshal(ValuesOverrideByte, &dat) - - valuesMap := make(map[string]map[string]interface{}) - valuesMap[appStoreAppVersion.AppStore.Name] = dat - valuesByte, err := json.Marshal(valuesMap) - if err != nil { - impl.logger.Errorw("error in marshaling", "err", err) - return installAppVersionRequest, nil, err - } - - valuesYamlConfig := &util.ChartConfig{ - FileName: appStoreBean.VALUES_YAML_FILE, - FileContent: string(valuesByte), - ChartName: chartMeta.Name, - ChartLocation: argocdAppName, - ChartRepoName: gitOpsRepoName, - ReleaseMessage: fmt.Sprintf("release-%d-env-%d ", appStoreAppVersion.Id, environment.Id), - UserEmailId: userEmailId, - UserName: userName, - } - - commitHash, _, err := impl.gitFactory.Client.CommitValues(valuesYamlConfig, gitOpsConfig) - if err != nil { - impl.logger.Errorw("error in git commit", "err", err) - //update timeline status for git commit failed state - gitCommitStatus := pipelineConfig.TIMELINE_STATUS_GIT_COMMIT_FAILED - gitCommitStatusDetail := fmt.Sprintf("Git commit failed - %v", err) - timeline := &pipelineConfig.PipelineStatusTimeline{ - InstalledAppVersionHistoryId: installAppVersionRequest.InstalledAppVersionHistoryId, - Status: gitCommitStatus, - StatusDetail: gitCommitStatusDetail, - StatusTime: time.Now(), - AuditLog: sql.AuditLog{ - CreatedBy: installAppVersionRequest.UserId, - CreatedOn: time.Now(), - UpdatedBy: installAppVersionRequest.UserId, - UpdatedOn: time.Now(), - }, - } - timelineErr := impl.pipelineStatusTimelineService.SaveTimeline(timeline, tx, true) - if timelineErr != nil { - impl.logger.Errorw("error in creating timeline status for git commit", "err", timelineErr, "timeline", timeline) - } - return installAppVersionRequest, nil, err - } - //creating timeline for Git Commit stage - timeline := &pipelineConfig.PipelineStatusTimeline{ - InstalledAppVersionHistoryId: installAppVersionRequest.InstalledAppVersionHistoryId, - Status: pipelineConfig.TIMELINE_STATUS_GIT_COMMIT, - StatusDetail: "Git commit done successfully.", - StatusTime: time.Now(), - AuditLog: sql.AuditLog{ - CreatedBy: installAppVersionRequest.UserId, - CreatedOn: time.Now(), - UpdatedBy: installAppVersionRequest.UserId, - UpdatedOn: time.Now(), - }, - } - err = impl.pipelineStatusTimelineService.SaveTimeline(timeline, tx, true) - if err != nil { - impl.logger.Errorw("error in creating timeline status for git commit", "err", err, "timeline", timeline) - } - - //sync local dir with remote - err = impl.chartTemplateService.GitPull(clonedDir, chartGitAttr.RepoUrl, appStoreName) - if err != nil { - impl.logger.Errorw("error in git pull", "err", err) - return installAppVersionRequest, nil, err - } - installAppVersionRequest.GitHash = commitHash - installAppVersionRequest.ACDAppName = argocdAppName - installAppVersionRequest.Environment = environment - - return installAppVersionRequest, chartGitAttr, nil -} - func (impl AppStoreDeploymentFullModeServiceImpl) AppStoreDeployOperationACD(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, chartGitAttr *util.ChartGitAttribute, ctx context.Context, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) { ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) defer cancel() @@ -378,18 +174,6 @@ func (impl AppStoreDeploymentFullModeServiceImpl) RegisterInArgo(chartGitAttribu return err } -func (impl AppStoreDeploymentFullModeServiceImpl) SyncACD(acdAppName string, ctx context.Context) { - req := new(application.ApplicationSyncRequest) - req.Name = &acdAppName - if ctx == nil { - impl.logger.Errorw("err in syncing ACD for AppStore, ctx is NULL", "acdAppName", acdAppName) - return - } - if _, err := impl.acdClient.Sync(ctx, req); err != nil { - impl.logger.Errorw("err in syncing ACD for AppStore", "acdAppName", acdAppName, "err", err) - } -} - func (impl AppStoreDeploymentFullModeServiceImpl) createInArgo(chartGitAttribute *util.ChartGitAttribute, ctx context.Context, envModel repository5.Environment, argocdAppName string) error { appNamespace := envModel.Namespace if appNamespace == "" { @@ -484,7 +268,7 @@ func (impl AppStoreDeploymentFullModeServiceImpl) UpdateRequirementYaml(installA return nil } -func (impl AppStoreDeploymentFullModeServiceImpl) SubscribeHelmInstallStatus() error { +func (impl AppStoreDeploymentFullModeServiceImpl) subscribeHelmInstallStatus() error { callback := func(msg *model.PubSubMsg) { diff --git a/pkg/appStore/deployment/service/AppStoreDeploymentService.go b/pkg/appStore/deployment/service/AppStoreDeploymentService.go index 02e55ca006..a06b2c76dc 100644 --- a/pkg/appStore/deployment/service/AppStoreDeploymentService.go +++ b/pkg/appStore/deployment/service/AppStoreDeploymentService.go @@ -34,10 +34,10 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" + repository3 "github.com/devtron-labs/devtron/pkg/appStore/chartGroup/repository" appStoreDeploymentCommon "github.com/devtron-labs/devtron/pkg/appStore/deployment/common" "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" appStoreDeploymentTool "github.com/devtron-labs/devtron/pkg/appStore/deployment/tool" - appStoreDeploymentGitopsTool "github.com/devtron-labs/devtron/pkg/appStore/deployment/tool/gitops" appStoreDiscoverRepository "github.com/devtron-labs/devtron/pkg/appStore/discover/repository" "github.com/devtron-labs/devtron/pkg/attributes" "github.com/devtron-labs/devtron/pkg/bean" @@ -91,18 +91,17 @@ func GetDeploymentServiceTypeConfig() (*DeploymentServiceTypeConfig, error) { type AppStoreDeploymentServiceImpl struct { logger *zap.SugaredLogger installedAppRepository repository.InstalledAppRepository - chartGroupDeploymentRepository repository.ChartGroupDeploymentRepository + chartGroupDeploymentRepository repository3.ChartGroupDeploymentRepository appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository environmentRepository clusterRepository.EnvironmentRepository clusterInstalledAppsRepository repository.ClusterInstalledAppsRepository appRepository app.AppRepository appStoreDeploymentHelmService appStoreDeploymentTool.AppStoreDeploymentHelmService - appStoreDeploymentArgoCdService appStoreDeploymentGitopsTool.AppStoreDeploymentArgoCdService + appStoreDeploymentArgoCdService appStoreDeploymentTool.AppStoreDeploymentArgoCdService environmentService cluster.EnvironmentService clusterService cluster.ClusterService helmAppService client.HelmAppService appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService - globalEnvVariables *util2.GlobalEnvVariables installedAppRepositoryHistory repository.InstalledAppVersionHistoryRepository gitOpsRepository repository2.GitOpsConfigRepository deploymentTypeConfig *DeploymentServiceTypeConfig @@ -111,12 +110,11 @@ type AppStoreDeploymentServiceImpl struct { } func NewAppStoreDeploymentServiceImpl(logger *zap.SugaredLogger, installedAppRepository repository.InstalledAppRepository, - chartGroupDeploymentRepository repository.ChartGroupDeploymentRepository, appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository, environmentRepository clusterRepository.EnvironmentRepository, + chartGroupDeploymentRepository repository3.ChartGroupDeploymentRepository, appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository, environmentRepository clusterRepository.EnvironmentRepository, clusterInstalledAppsRepository repository.ClusterInstalledAppsRepository, appRepository app.AppRepository, appStoreDeploymentHelmService appStoreDeploymentTool.AppStoreDeploymentHelmService, - appStoreDeploymentArgoCdService appStoreDeploymentGitopsTool.AppStoreDeploymentArgoCdService, environmentService cluster.EnvironmentService, + appStoreDeploymentArgoCdService appStoreDeploymentTool.AppStoreDeploymentArgoCdService, environmentService cluster.EnvironmentService, clusterService cluster.ClusterService, helmAppService client.HelmAppService, appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService, - globalEnvVariables *util2.GlobalEnvVariables, installedAppRepositoryHistory repository.InstalledAppVersionHistoryRepository, gitOpsRepository repository2.GitOpsConfigRepository, attributesService attributes.AttributesService, deploymentTypeConfig *DeploymentServiceTypeConfig, ChartTemplateService util.ChartTemplateService, aCDConfig *argocdServer.ACDConfig) *AppStoreDeploymentServiceImpl { @@ -134,7 +132,6 @@ func NewAppStoreDeploymentServiceImpl(logger *zap.SugaredLogger, installedAppRep clusterService: clusterService, helmAppService: helmAppService, appStoreDeploymentCommonService: appStoreDeploymentCommonService, - globalEnvVariables: globalEnvVariables, installedAppRepositoryHistory: installedAppRepositoryHistory, gitOpsRepository: gitOpsRepository, deploymentTypeConfig: deploymentTypeConfig, diff --git a/pkg/appStore/deployment/service/AppStoreDeploymentService_test.go b/pkg/appStore/deployment/service/AppStoreDeploymentService_test.go index bf7ff3567a..79a6fd1f9f 100644 --- a/pkg/appStore/deployment/service/AppStoreDeploymentService_test.go +++ b/pkg/appStore/deployment/service/AppStoreDeploymentService_test.go @@ -9,6 +9,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/util" appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" + repository5 "github.com/devtron-labs/devtron/pkg/appStore/chartGroup/repository" repository3 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" appStoreDiscoverRepository "github.com/devtron-labs/devtron/pkg/appStore/discover/repository" repository5 "github.com/devtron-labs/devtron/pkg/auth/user/repository" @@ -133,7 +134,7 @@ func initAppStoreDeploymentService(t *testing.T, internalUse bool) *AppStoreDepl db, _ := sql.NewDbConnection(config, sugaredLogger) gitOpsRepository := repository.NewGitOpsConfigRepositoryImpl(sugaredLogger, db) - chartGroupDeploymentRepository := repository3.NewChartGroupDeploymentRepositoryImpl(db, sugaredLogger) + chartGroupDeploymentRepository := repository5.NewChartGroupDeploymentRepositoryImpl(db, sugaredLogger) appStoreDiscoverRepository := appStoreDiscoverRepository.NewAppStoreApplicationVersionRepositoryImpl(sugaredLogger, db) diff --git a/pkg/appStore/deployment/service/InstalledAppService.go b/pkg/appStore/deployment/service/InstalledAppService.go index 58c54eb5a1..cc426a9381 100644 --- a/pkg/appStore/deployment/service/InstalledAppService.go +++ b/pkg/appStore/deployment/service/InstalledAppService.go @@ -27,32 +27,27 @@ import ( "io/ioutil" "net/http" "os" - "regexp" "strconv" "strings" - "sync" - "sync/atomic" "time" "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" "github.com/devtron-labs/common-lib/pubsub-lib/model" util4 "github.com/devtron-labs/common-lib/utils/k8s" - k8sCommonBean "github.com/devtron-labs/common-lib/utils/k8s/commonBean" - k8sObjectUtils "github.com/devtron-labs/common-lib/utils/k8sObjectsUtil" client "github.com/devtron-labs/devtron/api/helm-app" openapi "github.com/devtron-labs/devtron/api/helm-app/openapiClient" "github.com/devtron-labs/devtron/client/argocdServer" - "github.com/devtron-labs/devtron/internal/constants" "github.com/devtron-labs/devtron/internal/middleware" "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/pkg/app/status" "github.com/devtron-labs/devtron/pkg/appStatus" appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" + "github.com/devtron-labs/devtron/pkg/appStore/chartGroup" + repository6 "github.com/devtron-labs/devtron/pkg/appStore/chartGroup/repository" appStoreDeploymentCommon "github.com/devtron-labs/devtron/pkg/appStore/deployment/common" appStoreDeploymentFullMode "github.com/devtron-labs/devtron/pkg/appStore/deployment/fullMode" repository2 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" - appStoreDeploymentGitopsTool "github.com/devtron-labs/devtron/pkg/appStore/deployment/tool/gitops" appStoreDiscoverRepository "github.com/devtron-labs/devtron/pkg/appStore/discover/repository" "github.com/devtron-labs/devtron/pkg/appStore/values/service" "github.com/devtron-labs/devtron/pkg/auth/user" @@ -64,14 +59,12 @@ import ( util2 "github.com/devtron-labs/devtron/pkg/util" util3 "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/argo" - "github.com/tidwall/gjson" "github.com/Pallinder/go-randomdata" "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" pubsub "github.com/devtron-labs/common-lib/pubsub-lib" bean2 "github.com/devtron-labs/devtron/api/bean" application2 "github.com/devtron-labs/devtron/client/argocdServer/application" - "github.com/devtron-labs/devtron/client/argocdServer/repository" repository3 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/bean" @@ -88,29 +81,27 @@ const ( HELM_RELEASE_STATUS_UNKNOWN = "Unknown" ) +// DB operation + chart group + nats msg consume(to be removed) type InstalledAppService interface { GetAll(filter *appStoreBean.AppStoreFilter) (openapi.AppList, error) - DeployBulk(chartGroupInstallRequest *appStoreBean.ChartGroupInstallRequest) (*appStoreBean.ChartGroupInstallAppRes, error) - performDeployStage(appId int, installedAppVersionHistoryId int, userId int32) (*appStoreBean.InstallAppVersionDTO, error) + DeployBulk(chartGroupInstallRequest *chartGroup.ChartGroupInstallRequest) (*chartGroup.ChartGroupInstallAppRes, error) CheckAppExists(appNames []*appStoreBean.AppNames) ([]*appStoreBean.AppNames, error) DeployDefaultChartOnCluster(bean *cluster2.ClusterBean, userId int32) (bool, error) FindAppDetailsForAppstoreApplication(installedAppId, envId int) (bean2.AppDetailContainer, error) UpdateInstalledAppVersionStatus(application *v1alpha1.Application) (bool, error) - FetchResourceTree(rctx context.Context, cn http.CloseNotifier, appDetailsContainer *bean2.AppDetailsContainer, installedApp repository2.InstalledApps, helmReleaseInstallStatus string, status string) error MarkGitOpsInstalledAppsDeletedIfArgoAppIsDeleted(installedAppId int, envId int) error CheckAppExistsByInstalledAppId(installedAppId int) (*repository2.InstalledApps, error) - FindNotesForArgoApplication(installedAppId, envId int) (string, string, error) - FetchChartNotes(installedAppId int, envId int, token string, checkNotesAuth func(token string, appName string, envId int) bool) (string, error) + FetchResourceTreeWithHibernateForACD(rctx context.Context, cn http.CloseNotifier, appDetail *bean2.AppDetailContainer) bean2.AppDetailContainer - fetchResourceTreeForACD(rctx context.Context, cn http.CloseNotifier, appId int, envId, clusterId int, deploymentAppName, namespace string) (map[string]interface{}, error) + FetchResourceTree(rctx context.Context, cn http.CloseNotifier, appDetailsContainer *bean2.AppDetailsContainer, installedApp repository2.InstalledApps, helmReleaseInstallStatus string, status string) error + + //move to notes service + FetchChartNotes(installedAppId int, envId int, token string, checkNotesAuth func(token string, appName string, envId int) bool) (string, error) } type InstalledAppServiceImpl struct { logger *zap.SugaredLogger installedAppRepository repository2.InstalledAppRepository - chartTemplateService util.ChartTemplateService - refChartDir appStoreBean.RefChartProxyDir - repositoryService repository.ServiceClient appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository environmentRepository repository5.EnvironmentRepository teamRepository repository4.TeamRepository @@ -118,10 +109,8 @@ type InstalledAppServiceImpl struct { acdClient application2.ServiceClient appStoreValuesService service.AppStoreValuesService pubsubClient *pubsub.PubSubClientServiceImpl - tokenCache *util2.TokenCache - chartGroupDeploymentRepository repository2.ChartGroupDeploymentRepository + chartGroupDeploymentRepository repository6.ChartGroupDeploymentRepository envService cluster2.EnvironmentService - ArgoK8sClient argocdServer.ArgoK8sClient gitFactory *util.GitFactory aCDAuthConfig *util2.ACDAuthConfig gitOpsRepository repository3.GitOpsConfigRepository @@ -132,7 +121,6 @@ type InstalledAppServiceImpl struct { argoUserService argo.ArgoUserService helmAppClient client.HelmAppClient helmAppService client.HelmAppService - attributesRepository repository3.AttributesRepository appStatusService appStatus.AppStatusService K8sUtil *util4.K8sUtil pipelineStatusTimelineService status.PipelineStatusTimelineService @@ -144,34 +132,27 @@ type InstalledAppServiceImpl struct { func NewInstalledAppServiceImpl(logger *zap.SugaredLogger, installedAppRepository repository2.InstalledAppRepository, - chartTemplateService util.ChartTemplateService, refChartDir appStoreBean.RefChartProxyDir, - repositoryService repository.ServiceClient, appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository, environmentRepository repository5.EnvironmentRepository, teamRepository repository4.TeamRepository, appRepository app.AppRepository, acdClient application2.ServiceClient, appStoreValuesService service.AppStoreValuesService, pubsubClient *pubsub.PubSubClientServiceImpl, - tokenCache *util2.TokenCache, - chartGroupDeploymentRepository repository2.ChartGroupDeploymentRepository, - envService cluster2.EnvironmentService, argoK8sClient argocdServer.ArgoK8sClient, + chartGroupDeploymentRepository repository6.ChartGroupDeploymentRepository, + envService cluster2.EnvironmentService, gitFactory *util.GitFactory, aCDAuthConfig *util2.ACDAuthConfig, gitOpsRepository repository3.GitOpsConfigRepository, userService user.UserService, appStoreDeploymentFullModeService appStoreDeploymentFullMode.AppStoreDeploymentFullModeService, appStoreDeploymentService AppStoreDeploymentService, installedAppRepositoryHistory repository2.InstalledAppVersionHistoryRepository, argoUserService argo.ArgoUserService, helmAppClient client.HelmAppClient, helmAppService client.HelmAppService, - attributesRepository repository3.AttributesRepository, appStatusService appStatus.AppStatusService, K8sUtil *util4.K8sUtil, pipelineStatusTimelineService status.PipelineStatusTimelineService, appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService, - appStoreDeploymentArgoCdService appStoreDeploymentGitopsTool.AppStoreDeploymentArgoCdService, k8sCommonService k8s.K8sCommonService, k8sApplicationService application3.K8sApplicationService, + k8sCommonService k8s.K8sCommonService, k8sApplicationService application3.K8sApplicationService, acdConfig *argocdServer.ACDConfig) (*InstalledAppServiceImpl, error) { impl := &InstalledAppServiceImpl{ logger: logger, installedAppRepository: installedAppRepository, - chartTemplateService: chartTemplateService, - refChartDir: refChartDir, - repositoryService: repositoryService, appStoreApplicationVersionRepository: appStoreApplicationVersionRepository, environmentRepository: environmentRepository, teamRepository: teamRepository, @@ -179,10 +160,8 @@ func NewInstalledAppServiceImpl(logger *zap.SugaredLogger, acdClient: acdClient, appStoreValuesService: appStoreValuesService, pubsubClient: pubsubClient, - tokenCache: tokenCache, chartGroupDeploymentRepository: chartGroupDeploymentRepository, envService: envService, - ArgoK8sClient: argoK8sClient, gitFactory: gitFactory, aCDAuthConfig: aCDAuthConfig, gitOpsRepository: gitOpsRepository, @@ -193,7 +172,6 @@ func NewInstalledAppServiceImpl(logger *zap.SugaredLogger, argoUserService: argoUserService, helmAppClient: helmAppClient, helmAppService: helmAppService, - attributesRepository: attributesRepository, appStatusService: appStatusService, K8sUtil: K8sUtil, pipelineStatusTimelineService: pipelineStatusTimelineService, @@ -260,17 +238,6 @@ func (impl InstalledAppServiceImpl) GetAll(filter *appStoreBean.AppStoreFilter) return installedAppsResponse, nil } -// converts db object to bean -func (impl InstalledAppServiceImpl) chartAdaptor(chart *repository2.InstalledAppVersions) (*appStoreBean.InstallAppVersionDTO, error) { - - return &appStoreBean.InstallAppVersionDTO{ - InstalledAppId: chart.InstalledAppId, - Id: chart.Id, - AppStoreVersion: chart.AppStoreApplicationVersionId, - ValuesOverrideYaml: chart.ValuesYaml, - }, nil -} - func (impl InstalledAppServiceImpl) CheckAppExists(appNames []*appStoreBean.AppNames) ([]*appStoreBean.AppNames, error) { if len(appNames) == 0 { return nil, nil @@ -297,7 +264,7 @@ func (impl InstalledAppServiceImpl) CheckAppExists(appNames []*appStoreBean.AppN return appNames, nil } -func (impl InstalledAppServiceImpl) DeployBulk(chartGroupInstallRequest *appStoreBean.ChartGroupInstallRequest) (*appStoreBean.ChartGroupInstallAppRes, error) { +func (impl InstalledAppServiceImpl) DeployBulk(chartGroupInstallRequest *chartGroup.ChartGroupInstallRequest) (*chartGroup.ChartGroupInstallAppRes, error) { impl.logger.Debugw("bulk app install request", "req", chartGroupInstallRequest) //save in db // raise nats event @@ -349,7 +316,7 @@ func (impl InstalledAppServiceImpl) DeployBulk(chartGroupInstallRequest *appStor } //nats event impl.triggerDeploymentEvent(installAppVersions) - return &appStoreBean.ChartGroupInstallAppRes{}, nil + return &chartGroup.ChartGroupInstallAppRes{}, nil } // generate unique installation ID using APPID @@ -372,8 +339,8 @@ func (impl InstalledAppServiceImpl) getInstallationId(installAppVersions []*appS return fmt.Sprintf("%x", bs), nil } -func (impl InstalledAppServiceImpl) createChartGroupEntryObject(installAppVersionDTO *appStoreBean.InstallAppVersionDTO, chartGroupId int, groupINstallationId string) *repository2.ChartGroupDeployment { - return &repository2.ChartGroupDeployment{ +func (impl InstalledAppServiceImpl) createChartGroupEntryObject(installAppVersionDTO *appStoreBean.InstallAppVersionDTO, chartGroupId int, groupINstallationId string) *repository6.ChartGroupDeployment { + return &repository6.ChartGroupDeployment{ ChartGroupId: chartGroupId, ChartGroupEntryId: installAppVersionDTO.ChartGroupEntryId, InstalledAppId: installAppVersionDTO.InstalledAppId, @@ -591,7 +558,7 @@ func (impl InstalledAppServiceImpl) performDeployStage(installedAppVersionId int return installedAppVersion, nil } -func (impl InstalledAppServiceImpl) requestBuilderForBulkDeployment(installRequest *appStoreBean.ChartGroupInstallChartRequest, projectId int, userId int32) (*appStoreBean.InstallAppVersionDTO, error) { +func (impl InstalledAppServiceImpl) requestBuilderForBulkDeployment(installRequest *chartGroup.ChartGroupInstallChartRequest, projectId int, userId int32) (*appStoreBean.InstallAppVersionDTO, error) { valYaml := installRequest.ValuesOverrideYaml if valYaml == "" { valVersion, err := impl.appStoreValuesService.FindValuesByIdAndKind(installRequest.ReferenceValueId, installRequest.ReferenceValueKind) @@ -747,17 +714,17 @@ func (impl *InstalledAppServiceImpl) DeployDefaultChartOnCluster(bean *cluster2. // STEP 4 - prepare a bulk request (unique names need to apply for deploying chart) // STEP 4.1 - fetch chart for required name(actual chart name (app-store)) with default values // STEP 4.2 - update all the required charts, override values.yaml with env variables. - chartGroupInstallRequest := &appStoreBean.ChartGroupInstallRequest{} + chartGroupInstallRequest := &chartGroup.ChartGroupInstallRequest{} chartGroupInstallRequest.ProjectId = t.Id chartGroupInstallRequest.UserId = userId - var chartGroupInstallChartRequests []*appStoreBean.ChartGroupInstallChartRequest + var chartGroupInstallChartRequests []*chartGroup.ChartGroupInstallChartRequest for _, item := range charts.ChartComponent { appStore, err := impl.appStoreApplicationVersionRepository.FindByAppStoreName(item.Name) if err != nil { impl.logger.Errorw("DeployDefaultChartOnCluster, error in getting app store", "data", t, "err", err) return false, err } - chartGroupInstallChartRequest := &appStoreBean.ChartGroupInstallChartRequest{ + chartGroupInstallChartRequest := &chartGroup.ChartGroupInstallChartRequest{ AppName: fmt.Sprintf("%d-%d-%s", bean.Id, env.Id, item.Name), EnvironmentId: env.Id, ValuesOverrideYaml: item.Values, @@ -790,7 +757,7 @@ type ChartComponent struct { Values string `json:"values"` } -func (impl InstalledAppServiceImpl) DeployDefaultComponent(chartGroupInstallRequest *appStoreBean.ChartGroupInstallRequest) (*appStoreBean.ChartGroupInstallAppRes, error) { +func (impl InstalledAppServiceImpl) DeployDefaultComponent(chartGroupInstallRequest *chartGroup.ChartGroupInstallRequest) (*chartGroup.ChartGroupInstallAppRes, error) { impl.logger.Debugw("bulk app install request", "req", chartGroupInstallRequest) //save in db // raise nats event @@ -853,7 +820,7 @@ func (impl InstalledAppServiceImpl) DeployDefaultComponent(chartGroupInstallRequ } } - return &appStoreBean.ChartGroupInstallAppRes{}, nil + return &chartGroup.ChartGroupInstallAppRes{}, nil } func (impl *InstalledAppServiceImpl) FindAppDetailsForAppstoreApplication(installedAppId, envId int) (bean2.AppDetailContainer, error) { @@ -905,113 +872,6 @@ func (impl *InstalledAppServiceImpl) FindAppDetailsForAppstoreApplication(instal } return appDetail, nil } -func (impl *InstalledAppServiceImpl) FetchChartNotes(installedAppId int, envId int, token string, checkNotesAuth func(token string, appName string, envId int) bool) (string, error) { - //check notes.txt in db - installedApp, err := impl.installedAppRepository.FetchNotes(installedAppId) - if err != nil && err != pg.ErrNoRows { - return "", err - } - installedAppVerison, err := impl.installedAppRepository.GetInstalledAppVersionByInstalledAppIdAndEnvId(installedAppId, envId) - if err != nil { - if err == pg.ErrNoRows { - return "", fmt.Errorf("values are outdated. please fetch the latest version and try again") - } - impl.logger.Errorw("error fetching installed app version in installed app service", "err", err) - return "", err - } - chartVersion := installedAppVerison.AppStoreApplicationVersion.Version - if err != nil { - impl.logger.Errorw("error fetching chart version in installed app service", "err", err) - return "", err - } - re := regexp.MustCompile(`CHART VERSION: ([0-9]+\.[0-9]+\.[0-9]+)`) - newStr := re.ReplaceAllString(installedApp.Notes, "CHART VERSION: "+chartVersion) - installedApp.Notes = newStr - appName := installedApp.App.AppName - if err != nil { - impl.logger.Errorw("error fetching notes from db", "err", err) - return "", err - } - isValidAuth := checkNotesAuth(token, appName, envId) - if !isValidAuth { - impl.logger.Errorw("unauthorized user", "isValidAuth", isValidAuth) - return "", fmt.Errorf("unauthorized user") - } - //if notes is not present in db then below call will happen - if installedApp.Notes == "" { - notes, _, err := impl.FindNotesForArgoApplication(installedAppId, envId) - if err != nil { - impl.logger.Errorw("error fetching notes", "err", err) - return "", err - } - if notes == "" { - impl.logger.Errorw("error fetching notes", "err", err) - } - return notes, err - } - - return installedApp.Notes, nil -} -func (impl *InstalledAppServiceImpl) FindNotesForArgoApplication(installedAppId, envId int) (string, string, error) { - installedAppVerison, err := impl.installedAppRepository.GetInstalledAppVersionByInstalledAppIdAndEnvId(installedAppId, envId) - if err != nil { - impl.logger.Errorw("error fetching installed app version in installed app service", "err", err) - return "", "", err - } - var notes string - appName := installedAppVerison.InstalledApp.App.AppName - - if util.IsAcdApp(installedAppVerison.InstalledApp.DeploymentAppType) { - appStoreAppVersion, err := impl.appStoreApplicationVersionRepository.FindById(installedAppVerison.AppStoreApplicationVersion.Id) - if err != nil { - impl.logger.Errorw("error fetching app store app version in installed app service", "err", err) - return notes, appName, err - } - k8sServerVersion, err := impl.K8sUtil.GetKubeVersion() - if err != nil { - impl.logger.Errorw("exception caught in getting k8sServerVersion", "err", err) - return notes, appName, err - } - - installReleaseRequest := &client.InstallReleaseRequest{ - ChartName: appStoreAppVersion.Name, - ChartVersion: appStoreAppVersion.Version, - ValuesYaml: installedAppVerison.ValuesYaml, - K8SVersion: k8sServerVersion.String(), - ChartRepository: &client.ChartRepository{ - Name: appStoreAppVersion.AppStore.ChartRepo.Name, - Url: appStoreAppVersion.AppStore.ChartRepo.Url, - Username: appStoreAppVersion.AppStore.ChartRepo.UserName, - Password: appStoreAppVersion.AppStore.ChartRepo.Password, - }, - ReleaseIdentifier: &client.ReleaseIdentifier{ - ReleaseNamespace: installedAppVerison.InstalledApp.Environment.Namespace, - ReleaseName: installedAppVerison.InstalledApp.App.AppName, - }, - } - - clusterId := installedAppVerison.InstalledApp.Environment.ClusterId - config, err := impl.helmAppService.GetClusterConf(clusterId) - if err != nil { - impl.logger.Errorw("error in fetching cluster detail", "clusterId", clusterId, "err", err) - return "", appName, err - } - installReleaseRequest.ReleaseIdentifier.ClusterConfig = config - - notes, err = impl.helmAppService.GetNotes(context.Background(), installReleaseRequest) - if err != nil { - impl.logger.Errorw("error in fetching notes", "err", err) - return notes, appName, err - } - _, err = impl.appStoreDeploymentService.UpdateNotesForInstalledApp(installedAppId, notes) - if err != nil { - impl.logger.Errorw("error in updating notes in db ", "err", err) - return notes, appName, err - } - } - - return notes, appName, nil -} func (impl InstalledAppServiceImpl) GetInstalledAppVersionHistory(installedAppId int) (*appStoreBean.InstallAppVersionHistoryDto, error) { result := &appStoreBean.InstallAppVersionHistoryDto{} @@ -1121,82 +981,6 @@ func (impl InstalledAppServiceImpl) GetInstalledAppVersionHistoryValues(installe values.ValuesYaml = versionHistory.ValuesYamlRaw return values, err } -func (impl InstalledAppServiceImpl) FetchResourceTree(rctx context.Context, cn http.CloseNotifier, appDetailsContainer *bean2.AppDetailsContainer, installedApp repository2.InstalledApps, helmReleaseInstallStatus string, status string) error { - var err error - var resourceTree map[string]interface{} - deploymentAppName := fmt.Sprintf("%s-%s", installedApp.App.AppName, installedApp.Environment.Name) - if util.IsAcdApp(installedApp.DeploymentAppType) { - resourceTree, err = impl.fetchResourceTreeForACD(rctx, cn, installedApp.App.Id, installedApp.EnvironmentId, installedApp.Environment.ClusterId, deploymentAppName, installedApp.Environment.Namespace) - } else if util.IsHelmApp(installedApp.DeploymentAppType) { - config, err := impl.helmAppService.GetClusterConf(installedApp.Environment.ClusterId) - if err != nil { - impl.logger.Errorw("error in fetching cluster detail", "err", err) - } - req := &client.AppDetailRequest{ - ClusterConfig: config, - Namespace: installedApp.Environment.Namespace, - ReleaseName: installedApp.App.AppName, - } - detail, err := impl.helmAppClient.GetAppDetail(rctx, req) - if err != nil { - impl.logger.Errorw("error in fetching app detail", "err", err) - } - - /* helmReleaseInstallStatus is nats message sent from kubelink to orchestrator and has the following details about installation :- - 1) isReleaseInstalled -> whether release object is created or not in this installation - 2) ErrorInInstallation -> if there is error in installation - 3) Message -> error message/ success message - 4) InstallAppVersionHistoryId - 5) Status -> Progressing, Failed, Succeeded - */ - - if detail != nil && detail.ReleaseExist { - - resourceTree = util3.InterfaceToMapAdapter(detail.ResourceTreeResponse) - resourceTree["status"] = detail.ApplicationStatus - appDetailsContainer.Notes = detail.ChartMetadata.Notes - - helmInstallStatus := &appStoreBean.HelmReleaseStatusConfig{} - releaseStatus := detail.ReleaseStatus - - if len(helmReleaseInstallStatus) > 0 { - err := json.Unmarshal([]byte(helmReleaseInstallStatus), helmInstallStatus) - if err != nil { - impl.logger.Errorw("error in unmarshalling helm release install status") - return err - } - // ReleaseExist=true in app detail container but helm install status says that isReleaseInstalled=false which means this release was created externally - if helmInstallStatus.IsReleaseInstalled == false && status != "Progressing" { - /* - Handling case when :- - 1) An external release with name "foo" exist - 2) User creates an app with same name i.e "foo" - 3) In this case we use helmReleaseInstallStatus which will have status of our release and not external release - */ - resourceTree = make(map[string]interface{}) - releaseStatus = impl.getReleaseStatusFromHelmReleaseInstallStatus(helmReleaseInstallStatus, status) - } - } - releaseStatusMap := util3.InterfaceToMapAdapter(releaseStatus) - appDetailsContainer.ReleaseStatus = releaseStatusMap - } else { - // case when helm release is not created - releaseStatus := impl.getReleaseStatusFromHelmReleaseInstallStatus(helmReleaseInstallStatus, status) - releaseStatusMap := util3.InterfaceToMapAdapter(releaseStatus) - appDetailsContainer.ReleaseStatus = releaseStatusMap - } - } - if resourceTree != nil { - version, err := impl.k8sCommonService.GetK8sServerVersion(installedApp.Environment.ClusterId) - if err != nil { - impl.logger.Errorw("error in fetching k8s version in resource tree call fetching", "clusterId", installedApp.Environment.ClusterId, "err", err) - } else { - resourceTree["serverVersion"] = version.String() - } - appDetailsContainer.ResourceTree = resourceTree - } - return err -} func (impl InstalledAppServiceImpl) getReleaseStatusFromHelmReleaseInstallStatus(helmReleaseInstallStatus string, status string) *client.ReleaseStatus { //release status is sent in resource tree call and is shown on UI as helm config apply status @@ -1291,233 +1075,3 @@ func (impl InstalledAppServiceImpl) CheckAppExistsByInstalledAppId(installedAppI } return installedApp, err } - -func (impl InstalledAppServiceImpl) FetchResourceTreeWithHibernateForACD(rctx context.Context, cn http.CloseNotifier, appDetail *bean2.AppDetailContainer) bean2.AppDetailContainer { - ctx, cancel := context.WithCancel(rctx) - if cn != nil { - go func(done <-chan struct{}, closed <-chan bool) { - select { - case <-done: - case <-closed: - cancel() - } - }(ctx.Done(), cn.CloseNotify()) - } - acdToken, err := impl.argoUserService.GetLatestDevtronArgoCdUserToken() - if err != nil { - impl.logger.Errorw("error in getting acd token", "err", err) - return *appDetail - } - ctx = context.WithValue(ctx, "token", acdToken) - defer cancel() - deploymentAppName := fmt.Sprintf("%s-%s", appDetail.AppName, appDetail.EnvironmentName) - resourceTree, err := impl.fetchResourceTreeForACD(rctx, cn, appDetail.InstalledAppId, appDetail.EnvironmentId, appDetail.ClusterId, deploymentAppName, appDetail.Namespace) - appDetail.ResourceTree = resourceTree - if err != nil { - return *appDetail - } - if appDetail.ResourceTree["nodes"] == nil { - return *appDetail - } - appDetail.ResourceTree, _ = impl.checkHibernate(appDetail.ResourceTree, deploymentAppName, ctx) - return *appDetail -} -func (impl InstalledAppServiceImpl) checkHibernate(resp map[string]interface{}, deploymentAppName string, ctx context.Context) (map[string]interface{}, string) { - - if resp == nil { - return resp, "" - } - responseTree := resp - var canBeHibernated uint64 = 0 - var hibernated uint64 = 0 - responseTreeNodes, ok := responseTree["nodes"] - if !ok { - return resp, "" - } - replicaNodes := impl.filterOutReplicaNodes(responseTreeNodes) - batchSize := impl.aCDAuthConfig.ResourceListForReplicasBatchSize - requestsLength := len(replicaNodes) - for i := 0; i < requestsLength; { - //requests left to process - remainingBatch := requestsLength - i - if remainingBatch < batchSize { - batchSize = remainingBatch - } - var wg sync.WaitGroup - for j := 0; j < batchSize; j++ { - wg.Add(1) - go func(j int) { - defer wg.Done() - canBeHibernatedFlag, hibernatedFlag := impl.processReplicaNodeForHibernation(replicaNodes[i+j], deploymentAppName, ctx) - if canBeHibernatedFlag { - atomic.AddUint64(&canBeHibernated, 1) - } - if hibernatedFlag { - atomic.AddUint64(&hibernated, 1) - } - }(j) - } - wg.Wait() - i += batchSize - } - - status := "" - if hibernated > 0 && canBeHibernated > 0 { - if hibernated == canBeHibernated { - status = appStatus.HealthStatusHibernating - } else if hibernated < canBeHibernated { - status = appStatus.HealthStatusPartiallyHibernated - } - } - - return responseTree, status -} - -func (impl InstalledAppServiceImpl) processReplicaNodeForHibernation(node interface{}, deploymentAppName string, ctx context.Context) (bool, bool) { - currNode := node.(interface{}).(map[string]interface{}) - resName := util3.InterfaceToString(currNode["name"]) - resKind := util3.InterfaceToString(currNode["kind"]) - resGroup := util3.InterfaceToString(currNode["group"]) - resVersion := util3.InterfaceToString(currNode["version"]) - resNamespace := util3.InterfaceToString(currNode["namespace"]) - rQuery := &application.ApplicationResourceRequest{ - Name: &deploymentAppName, - ResourceName: &resName, - Kind: &resKind, - Group: &resGroup, - Version: &resVersion, - Namespace: &resNamespace, - } - canBeHibernatedFlag := false - alreadyHibernated := false - - if currNode["parentRefs"] == nil { - canBeHibernatedFlag, alreadyHibernated = impl.checkForHibernation(ctx, rQuery, currNode) - } - return canBeHibernatedFlag, alreadyHibernated -} - -func (impl InstalledAppServiceImpl) checkForHibernation(ctx context.Context, rQuery *application.ApplicationResourceRequest, currNode map[string]interface{}) (bool, bool) { - t0 := time.Now() - canBeHibernated := false - alreadyHibernated := false - ctx, _ = context.WithTimeout(ctx, 60*time.Second) - res, err := impl.acdClient.GetResource(ctx, rQuery) - if err != nil { - impl.logger.Errorw("error getting response from acdClient", "request", rQuery, "data", res, "timeTaken", time.Since(t0), "err", err) - return canBeHibernated, alreadyHibernated - } - if res.Manifest != nil { - manifest, _ := gjson.Parse(*res.Manifest).Value().(map[string]interface{}) - replicas := util3.InterfaceToMapAdapter(manifest["spec"])["replicas"] - if replicas != nil { - currNode["canBeHibernated"] = true - canBeHibernated = true - } - annotations := util3.InterfaceToMapAdapter(manifest["metadata"])["annotations"] - if annotations != nil { - val := util3.InterfaceToMapAdapter(annotations)["hibernator.devtron.ai/replicas"] - if val != nil { - if util3.InterfaceToString(val) != "0" && util3.InterfaceToFloat(replicas) == 0 { - currNode["isHibernated"] = true - alreadyHibernated = true - } - } - } - } - return canBeHibernated, alreadyHibernated -} - -func (impl InstalledAppServiceImpl) fetchResourceTreeForACD(rctx context.Context, cn http.CloseNotifier, appId int, envId, clusterId int, deploymentAppName, namespace string) (map[string]interface{}, error) { - var resourceTree map[string]interface{} - query := &application.ResourcesQuery{ - ApplicationName: &deploymentAppName, - } - ctx, cancel := context.WithCancel(rctx) - if cn != nil { - go func(done <-chan struct{}, closed <-chan bool) { - select { - case <-done: - case <-closed: - cancel() - } - }(ctx.Done(), cn.CloseNotify()) - } - acdToken, err := impl.argoUserService.GetLatestDevtronArgoCdUserToken() - if err != nil { - impl.logger.Errorw("error in getting acd token", "err", err) - return resourceTree, err - } - ctx = context.WithValue(ctx, "token", acdToken) - defer cancel() - start := time.Now() - resp, err := impl.acdClient.ResourceTree(ctx, query) - elapsed := time.Since(start) - impl.logger.Debugf("Time elapsed %s in fetching app-store installed application %s for environment %s", elapsed, deploymentAppName, envId) - if err != nil { - impl.logger.Errorw("service err, FetchAppDetailsForInstalledAppV2, fetching resource tree", "err", err, "installedAppId", appId, "envId", envId) - err = &util.ApiError{ - Code: constants.AppDetailResourceTreeNotFound, - InternalMessage: "app detail fetched, failed to get resource tree from acd", - UserMessage: "app detail fetched, failed to get resource tree from acd", - } - return resourceTree, err - } - label := fmt.Sprintf("app.kubernetes.io/instance=%s", deploymentAppName) - pods, err := impl.k8sApplicationService.GetPodListByLabel(clusterId, namespace, label) - if err != nil { - impl.logger.Errorw("error in getting pods by label", "err", err, "clusterId", clusterId, "namespace", namespace, "label", label) - return resourceTree, err - } - ephemeralContainersMap := k8sObjectUtils.ExtractEphemeralContainers(pods) - for _, metaData := range resp.PodMetadata { - metaData.EphemeralContainers = ephemeralContainersMap[metaData.Name] - } - resourceTree = util3.InterfaceToMapAdapter(resp) - resourceTree, hibernationStatus := impl.checkHibernate(resourceTree, deploymentAppName, ctx) - appStatus := resp.Status - if resourceTree != nil { - if hibernationStatus != "" { - resourceTree["status"] = hibernationStatus - appStatus = hibernationStatus - } - } - // using this resp.Status to update in app_status table - go func() { - err = impl.appStatusService.UpdateStatusWithAppIdEnvId(appId, envId, appStatus) - if err != nil { - impl.logger.Warnw("error in updating app status", "err", err, appId, "envId", envId) - } - }() - impl.logger.Debugf("application %s in environment %s had status %+v\n", appId, envId, resp) - k8sAppDetail := bean2.AppDetailContainer{ - DeploymentDetailContainer: bean2.DeploymentDetailContainer{ - ClusterId: clusterId, - Namespace: namespace, - }, - } - clusterIdString := strconv.Itoa(clusterId) - validRequest := impl.k8sCommonService.FilterK8sResources(rctx, resourceTree, k8sAppDetail, clusterIdString, []string{k8sCommonBean.ServiceKind, k8sCommonBean.EndpointsKind, k8sCommonBean.IngressKind}) - response, err := impl.k8sCommonService.GetManifestsByBatch(rctx, validRequest) - if err != nil { - impl.logger.Errorw("error in getting manifest by batch", "err", err, "clusterId", clusterIdString) - return nil, err - } - newResourceTree := impl.k8sCommonService.PortNumberExtraction(response, resourceTree) - return newResourceTree, err -} - -func (impl InstalledAppServiceImpl) filterOutReplicaNodes(responseTreeNodes interface{}) []interface{} { - resourceListForReplicas := impl.aCDAuthConfig.ResourceListForReplicas - entries := strings.Split(resourceListForReplicas, ",") - resourceListMap := util3.ConvertStringSliceToMap(entries) - var replicaNodes []interface{} - for _, node := range responseTreeNodes.(interface{}).([]interface{}) { - currNode := node.(interface{}).(map[string]interface{}) - resKind := util3.InterfaceToString(currNode["kind"]) - if _, ok := resourceListMap[resKind]; ok { - replicaNodes = append(replicaNodes, node) - } - } - return replicaNodes -} diff --git a/pkg/appStore/deployment/service/InstalledAppService_test.go b/pkg/appStore/deployment/service/InstalledAppService_test.go index 7de0bb6574..b28d642ec6 100644 --- a/pkg/appStore/deployment/service/InstalledAppService_test.go +++ b/pkg/appStore/deployment/service/InstalledAppService_test.go @@ -11,6 +11,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/util" appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" + repository5 "github.com/devtron-labs/devtron/pkg/appStore/chartGroup/repository" appStoreDeploymentFullMode "github.com/devtron-labs/devtron/pkg/appStore/deployment/fullMode" repository4 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" appStoreDiscoverRepository "github.com/devtron-labs/devtron/pkg/appStore/discover/repository" @@ -38,7 +39,7 @@ func TestInstalledAppServiceImpl_DeployDefaultChartOnCluster(t *testing.T) { appStoreValuesService service.AppStoreValuesService pubsubClient *pubsub.PubSubClientServiceImpl tokenCache *util2.TokenCache - chartGroupDeploymentRepository repository4.ChartGroupDeploymentRepository + chartGroupDeploymentRepository repository5.ChartGroupDeploymentRepository envService cluster.EnvironmentService ArgoK8sClient argocdServer.ArgoK8sClient gitFactory *util.GitFactory diff --git a/pkg/appStore/deployment/service/Notes.go b/pkg/appStore/deployment/service/Notes.go new file mode 100644 index 0000000000..88ce454af6 --- /dev/null +++ b/pkg/appStore/deployment/service/Notes.go @@ -0,0 +1,118 @@ +package service + +import ( + "context" + "fmt" + "github.com/devtron-labs/devtron/api/helm-app" + "github.com/devtron-labs/devtron/internal/util" + "github.com/go-pg/pg" + "regexp" +) + +func (impl *InstalledAppServiceImpl) findNotesForArgoApplication(installedAppId, envId int) (string, string, error) { + installedAppVerison, err := impl.installedAppRepository.GetInstalledAppVersionByInstalledAppIdAndEnvId(installedAppId, envId) + if err != nil { + impl.logger.Errorw("error fetching installed app version in installed app service", "err", err) + return "", "", err + } + var notes string + appName := installedAppVerison.InstalledApp.App.AppName + + if util.IsAcdApp(installedAppVerison.InstalledApp.DeploymentAppType) { + appStoreAppVersion, err := impl.appStoreApplicationVersionRepository.FindById(installedAppVerison.AppStoreApplicationVersion.Id) + if err != nil { + impl.logger.Errorw("error fetching app store app version in installed app service", "err", err) + return notes, appName, err + } + k8sServerVersion, err := impl.K8sUtil.GetKubeVersion() + if err != nil { + impl.logger.Errorw("exception caught in getting k8sServerVersion", "err", err) + return notes, appName, err + } + + installReleaseRequest := &client.InstallReleaseRequest{ + ChartName: appStoreAppVersion.Name, + ChartVersion: appStoreAppVersion.Version, + ValuesYaml: installedAppVerison.ValuesYaml, + K8SVersion: k8sServerVersion.String(), + ChartRepository: &client.ChartRepository{ + Name: appStoreAppVersion.AppStore.ChartRepo.Name, + Url: appStoreAppVersion.AppStore.ChartRepo.Url, + Username: appStoreAppVersion.AppStore.ChartRepo.UserName, + Password: appStoreAppVersion.AppStore.ChartRepo.Password, + }, + ReleaseIdentifier: &client.ReleaseIdentifier{ + ReleaseNamespace: installedAppVerison.InstalledApp.Environment.Namespace, + ReleaseName: installedAppVerison.InstalledApp.App.AppName, + }, + } + + clusterId := installedAppVerison.InstalledApp.Environment.ClusterId + config, err := impl.helmAppService.GetClusterConf(clusterId) + if err != nil { + impl.logger.Errorw("error in fetching cluster detail", "clusterId", clusterId, "err", err) + return "", appName, err + } + installReleaseRequest.ReleaseIdentifier.ClusterConfig = config + + notes, err = impl.helmAppService.GetNotes(context.Background(), installReleaseRequest) + if err != nil { + impl.logger.Errorw("error in fetching notes", "err", err) + return notes, appName, err + } + _, err = impl.appStoreDeploymentService.UpdateNotesForInstalledApp(installedAppId, notes) + if err != nil { + impl.logger.Errorw("error in updating notes in db ", "err", err) + return notes, appName, err + } + } + + return notes, appName, nil +} +func (impl *InstalledAppServiceImpl) FetchChartNotes(installedAppId int, envId int, token string, checkNotesAuth func(token string, appName string, envId int) bool) (string, error) { + //check notes.txt in db + installedApp, err := impl.installedAppRepository.FetchNotes(installedAppId) + if err != nil && err != pg.ErrNoRows { + return "", err + } + installedAppVerison, err := impl.installedAppRepository.GetInstalledAppVersionByInstalledAppIdAndEnvId(installedAppId, envId) + if err != nil { + if err == pg.ErrNoRows { + return "", fmt.Errorf("values are outdated. please fetch the latest version and try again") + } + impl.logger.Errorw("error fetching installed app version in installed app service", "err", err) + return "", err + } + chartVersion := installedAppVerison.AppStoreApplicationVersion.Version + if err != nil { + impl.logger.Errorw("error fetching chart version in installed app service", "err", err) + return "", err + } + re := regexp.MustCompile(`CHART VERSION: ([0-9]+\.[0-9]+\.[0-9]+)`) + newStr := re.ReplaceAllString(installedApp.Notes, "CHART VERSION: "+chartVersion) + installedApp.Notes = newStr + appName := installedApp.App.AppName + if err != nil { + impl.logger.Errorw("error fetching notes from db", "err", err) + return "", err + } + isValidAuth := checkNotesAuth(token, appName, envId) + if !isValidAuth { + impl.logger.Errorw("unauthorized user", "isValidAuth", isValidAuth) + return "", fmt.Errorf("unauthorized user") + } + //if notes is not present in db then below call will happen + if installedApp.Notes == "" { + notes, _, err := impl.findNotesForArgoApplication(installedAppId, envId) + if err != nil { + impl.logger.Errorw("error fetching notes", "err", err) + return "", err + } + if notes == "" { + impl.logger.Errorw("error fetching notes", "err", err) + } + return notes, err + } + + return installedApp.Notes, nil +} diff --git a/pkg/appStore/deployment/service/ResourceTree.go b/pkg/appStore/deployment/service/ResourceTree.go new file mode 100644 index 0000000000..e826bacb55 --- /dev/null +++ b/pkg/appStore/deployment/service/ResourceTree.go @@ -0,0 +1,334 @@ +package service + +import ( + "context" + "encoding/json" + "fmt" + "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" + "github.com/devtron-labs/common-lib/utils/k8s/commonBean" + "github.com/devtron-labs/common-lib/utils/k8sObjectsUtil" + "github.com/devtron-labs/devtron/api/bean" + "github.com/devtron-labs/devtron/api/helm-app" + "github.com/devtron-labs/devtron/internal/constants" + "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/appStatus" + "github.com/devtron-labs/devtron/pkg/appStore/bean" + "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" + util2 "github.com/devtron-labs/devtron/util" + "github.com/tidwall/gjson" + "net/http" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" +) + +func (impl InstalledAppServiceImpl) FetchResourceTree(rctx context.Context, cn http.CloseNotifier, appDetailsContainer *bean.AppDetailsContainer, installedApp repository.InstalledApps, helmReleaseInstallStatus string, status string) error { + var err error + var resourceTree map[string]interface{} + deploymentAppName := fmt.Sprintf("%s-%s", installedApp.App.AppName, installedApp.Environment.Name) + if util.IsAcdApp(installedApp.DeploymentAppType) { + resourceTree, err = impl.fetchResourceTreeForACD(rctx, cn, installedApp.App.Id, installedApp.EnvironmentId, installedApp.Environment.ClusterId, deploymentAppName, installedApp.Environment.Namespace) + } else if util.IsHelmApp(installedApp.DeploymentAppType) { + config, err := impl.helmAppService.GetClusterConf(installedApp.Environment.ClusterId) + if err != nil { + impl.logger.Errorw("error in fetching cluster detail", "err", err) + } + req := &client.AppDetailRequest{ + ClusterConfig: config, + Namespace: installedApp.Environment.Namespace, + ReleaseName: installedApp.App.AppName, + } + detail, err := impl.helmAppClient.GetAppDetail(rctx, req) + if err != nil { + impl.logger.Errorw("error in fetching app detail", "err", err) + } + + /* helmReleaseInstallStatus is nats message sent from kubelink to orchestrator and has the following details about installation :- + 1) isReleaseInstalled -> whether release object is created or not in this installation + 2) ErrorInInstallation -> if there is error in installation + 3) Message -> error message/ success message + 4) InstallAppVersionHistoryId + 5) Status -> Progressing, Failed, Succeeded + */ + + if detail != nil && detail.ReleaseExist { + + resourceTree = util2.InterfaceToMapAdapter(detail.ResourceTreeResponse) + resourceTree["status"] = detail.ApplicationStatus + appDetailsContainer.Notes = detail.ChartMetadata.Notes + + helmInstallStatus := &appStoreBean.HelmReleaseStatusConfig{} + releaseStatus := detail.ReleaseStatus + + if len(helmReleaseInstallStatus) > 0 { + err := json.Unmarshal([]byte(helmReleaseInstallStatus), helmInstallStatus) + if err != nil { + impl.logger.Errorw("error in unmarshalling helm release install status") + return err + } + // ReleaseExist=true in app detail container but helm install status says that isReleaseInstalled=false which means this release was created externally + if helmInstallStatus.IsReleaseInstalled == false && status != "Progressing" { + /* + Handling case when :- + 1) An external release with name "foo" exist + 2) User creates an app with same name i.e "foo" + 3) In this case we use helmReleaseInstallStatus which will have status of our release and not external release + */ + resourceTree = make(map[string]interface{}) + releaseStatus = impl.getReleaseStatusFromHelmReleaseInstallStatus(helmReleaseInstallStatus, status) + } + } + releaseStatusMap := util2.InterfaceToMapAdapter(releaseStatus) + appDetailsContainer.ReleaseStatus = releaseStatusMap + } else { + // case when helm release is not created + releaseStatus := impl.getReleaseStatusFromHelmReleaseInstallStatus(helmReleaseInstallStatus, status) + releaseStatusMap := util2.InterfaceToMapAdapter(releaseStatus) + appDetailsContainer.ReleaseStatus = releaseStatusMap + } + } + if resourceTree != nil { + version, err := impl.k8sCommonService.GetK8sServerVersion(installedApp.Environment.ClusterId) + if err != nil { + impl.logger.Errorw("error in fetching k8s version in resource tree call fetching", "clusterId", installedApp.Environment.ClusterId, "err", err) + } else { + resourceTree["serverVersion"] = version.String() + } + appDetailsContainer.ResourceTree = resourceTree + } + return err +} + +func (impl InstalledAppServiceImpl) fetchResourceTreeForACD(rctx context.Context, cn http.CloseNotifier, appId int, envId, clusterId int, deploymentAppName, namespace string) (map[string]interface{}, error) { + var resourceTree map[string]interface{} + query := &application.ResourcesQuery{ + ApplicationName: &deploymentAppName, + } + ctx, cancel := context.WithCancel(rctx) + if cn != nil { + go func(done <-chan struct{}, closed <-chan bool) { + select { + case <-done: + case <-closed: + cancel() + } + }(ctx.Done(), cn.CloseNotify()) + } + acdToken, err := impl.argoUserService.GetLatestDevtronArgoCdUserToken() + if err != nil { + impl.logger.Errorw("error in getting acd token", "err", err) + return resourceTree, err + } + ctx = context.WithValue(ctx, "token", acdToken) + defer cancel() + start := time.Now() + resp, err := impl.acdClient.ResourceTree(ctx, query) + elapsed := time.Since(start) + impl.logger.Debugf("Time elapsed %s in fetching app-store installed application %s for environment %s", elapsed, deploymentAppName, envId) + if err != nil { + impl.logger.Errorw("service err, FetchAppDetailsForInstalledAppV2, fetching resource tree", "err", err, "installedAppId", appId, "envId", envId) + err = &util.ApiError{ + Code: constants.AppDetailResourceTreeNotFound, + InternalMessage: "app detail fetched, failed to get resource tree from acd", + UserMessage: "app detail fetched, failed to get resource tree from acd", + } + return resourceTree, err + } + label := fmt.Sprintf("app.kubernetes.io/instance=%s", deploymentAppName) + pods, err := impl.k8sApplicationService.GetPodListByLabel(clusterId, namespace, label) + if err != nil { + impl.logger.Errorw("error in getting pods by label", "err", err, "clusterId", clusterId, "namespace", namespace, "label", label) + return resourceTree, err + } + ephemeralContainersMap := k8sObjectsUtil.ExtractEphemeralContainers(pods) + for _, metaData := range resp.PodMetadata { + metaData.EphemeralContainers = ephemeralContainersMap[metaData.Name] + } + resourceTree = util2.InterfaceToMapAdapter(resp) + resourceTree, hibernationStatus := impl.checkHibernate(resourceTree, deploymentAppName, ctx) + appStatus := resp.Status + if resourceTree != nil { + if hibernationStatus != "" { + resourceTree["status"] = hibernationStatus + appStatus = hibernationStatus + } + } + // using this resp.Status to update in app_status table + //FIXME: remove this dangling thread + go func() { + err = impl.appStatusService.UpdateStatusWithAppIdEnvId(appId, envId, appStatus) + if err != nil { + impl.logger.Warnw("error in updating app status", "err", err, appId, "envId", envId) + } + }() + impl.logger.Debugf("application %s in environment %s had status %+v\n", appId, envId, resp) + k8sAppDetail := bean.AppDetailContainer{ + DeploymentDetailContainer: bean.DeploymentDetailContainer{ + ClusterId: clusterId, + Namespace: namespace, + }, + } + clusterIdString := strconv.Itoa(clusterId) + validRequest := impl.k8sCommonService.FilterK8sResources(rctx, resourceTree, k8sAppDetail, clusterIdString, []string{commonBean.ServiceKind, commonBean.EndpointsKind, commonBean.IngressKind}) + response, err := impl.k8sCommonService.GetManifestsByBatch(rctx, validRequest) + if err != nil { + impl.logger.Errorw("error in getting manifest by batch", "err", err, "clusterId", clusterIdString) + return nil, err + } + newResourceTree := impl.k8sCommonService.PortNumberExtraction(response, resourceTree) + return newResourceTree, err +} + +func (impl InstalledAppServiceImpl) FetchResourceTreeWithHibernateForACD(rctx context.Context, cn http.CloseNotifier, appDetail *bean.AppDetailContainer) bean.AppDetailContainer { + ctx, cancel := context.WithCancel(rctx) + if cn != nil { + go func(done <-chan struct{}, closed <-chan bool) { + select { + case <-done: + case <-closed: + cancel() + } + }(ctx.Done(), cn.CloseNotify()) + } + acdToken, err := impl.argoUserService.GetLatestDevtronArgoCdUserToken() + if err != nil { + impl.logger.Errorw("error in getting acd token", "err", err) + return *appDetail + } + ctx = context.WithValue(ctx, "token", acdToken) + defer cancel() + deploymentAppName := fmt.Sprintf("%s-%s", appDetail.AppName, appDetail.EnvironmentName) + resourceTree, err := impl.fetchResourceTreeForACD(rctx, cn, appDetail.InstalledAppId, appDetail.EnvironmentId, appDetail.ClusterId, deploymentAppName, appDetail.Namespace) + appDetail.ResourceTree = resourceTree + if err != nil { + return *appDetail + } + if appDetail.ResourceTree["nodes"] == nil { + return *appDetail + } + appDetail.ResourceTree, _ = impl.checkHibernate(appDetail.ResourceTree, deploymentAppName, ctx) + return *appDetail +} + +func (impl InstalledAppServiceImpl) checkHibernate(resp map[string]interface{}, deploymentAppName string, ctx context.Context) (map[string]interface{}, string) { + + if resp == nil { + return resp, "" + } + responseTree := resp + var canBeHibernated uint64 = 0 + var hibernated uint64 = 0 + responseTreeNodes, ok := responseTree["nodes"] + if !ok { + return resp, "" + } + replicaNodes := impl.filterOutReplicaNodes(responseTreeNodes) + batchSize := impl.aCDAuthConfig.ResourceListForReplicasBatchSize + requestsLength := len(replicaNodes) + for i := 0; i < requestsLength; { + //requests left to process + remainingBatch := requestsLength - i + if remainingBatch < batchSize { + batchSize = remainingBatch + } + var wg sync.WaitGroup + for j := 0; j < batchSize; j++ { + wg.Add(1) + go func(j int) { + defer wg.Done() + canBeHibernatedFlag, hibernatedFlag := impl.processReplicaNodeForHibernation(replicaNodes[i+j], deploymentAppName, ctx) + if canBeHibernatedFlag { + atomic.AddUint64(&canBeHibernated, 1) + } + if hibernatedFlag { + atomic.AddUint64(&hibernated, 1) + } + }(j) + } + wg.Wait() + i += batchSize + } + + status := "" + if hibernated > 0 && canBeHibernated > 0 { + if hibernated == canBeHibernated { + status = appStatus.HealthStatusHibernating + } else if hibernated < canBeHibernated { + status = appStatus.HealthStatusPartiallyHibernated + } + } + + return responseTree, status +} + +func (impl InstalledAppServiceImpl) processReplicaNodeForHibernation(node interface{}, deploymentAppName string, ctx context.Context) (bool, bool) { + currNode := node.(interface{}).(map[string]interface{}) + resName := util2.InterfaceToString(currNode["name"]) + resKind := util2.InterfaceToString(currNode["kind"]) + resGroup := util2.InterfaceToString(currNode["group"]) + resVersion := util2.InterfaceToString(currNode["version"]) + resNamespace := util2.InterfaceToString(currNode["namespace"]) + rQuery := &application.ApplicationResourceRequest{ + Name: &deploymentAppName, + ResourceName: &resName, + Kind: &resKind, + Group: &resGroup, + Version: &resVersion, + Namespace: &resNamespace, + } + canBeHibernatedFlag := false + alreadyHibernated := false + + if currNode["parentRefs"] == nil { + canBeHibernatedFlag, alreadyHibernated = impl.checkForHibernation(ctx, rQuery, currNode) + } + return canBeHibernatedFlag, alreadyHibernated +} + +func (impl InstalledAppServiceImpl) checkForHibernation(ctx context.Context, rQuery *application.ApplicationResourceRequest, currNode map[string]interface{}) (bool, bool) { + t0 := time.Now() + canBeHibernated := false + alreadyHibernated := false + ctx, _ = context.WithTimeout(ctx, 60*time.Second) + res, err := impl.acdClient.GetResource(ctx, rQuery) + if err != nil { + impl.logger.Errorw("error getting response from acdClient", "request", rQuery, "data", res, "timeTaken", time.Since(t0), "err", err) + return canBeHibernated, alreadyHibernated + } + if res.Manifest != nil { + manifest, _ := gjson.Parse(*res.Manifest).Value().(map[string]interface{}) + replicas := util2.InterfaceToMapAdapter(manifest["spec"])["replicas"] + if replicas != nil { + currNode["canBeHibernated"] = true + canBeHibernated = true + } + annotations := util2.InterfaceToMapAdapter(manifest["metadata"])["annotations"] + if annotations != nil { + val := util2.InterfaceToMapAdapter(annotations)["hibernator.devtron.ai/replicas"] + if val != nil { + if util2.InterfaceToString(val) != "0" && util2.InterfaceToFloat(replicas) == 0 { + currNode["isHibernated"] = true + alreadyHibernated = true + } + } + } + } + return canBeHibernated, alreadyHibernated +} + +func (impl InstalledAppServiceImpl) filterOutReplicaNodes(responseTreeNodes interface{}) []interface{} { + resourceListForReplicas := impl.aCDAuthConfig.ResourceListForReplicas + entries := strings.Split(resourceListForReplicas, ",") + resourceListMap := util2.ConvertStringSliceToMap(entries) + var replicaNodes []interface{} + for _, node := range responseTreeNodes.(interface{}).([]interface{}) { + currNode := node.(interface{}).(map[string]interface{}) + resKind := util2.InterfaceToString(currNode["kind"]) + if _, ok := resourceListMap[resKind]; ok { + replicaNodes = append(replicaNodes, node) + } + } + return replicaNodes +} diff --git a/pkg/appStore/deployment/tool/gitops/AppStoreDeploymentArgoCdService.go b/pkg/appStore/deployment/tool/AppStoreDeploymentArgoCdService.go similarity index 66% rename from pkg/appStore/deployment/tool/gitops/AppStoreDeploymentArgoCdService.go rename to pkg/appStore/deployment/tool/AppStoreDeploymentArgoCdService.go index c861291f38..e92a5e0485 100644 --- a/pkg/appStore/deployment/tool/gitops/AppStoreDeploymentArgoCdService.go +++ b/pkg/appStore/deployment/tool/AppStoreDeploymentArgoCdService.go @@ -1,29 +1,26 @@ -package appStoreDeploymentGitopsTool +package appStoreDeploymentTool import ( "context" "encoding/json" "errors" "fmt" - "net/http" "strings" "time" "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" - "github.com/devtron-labs/common-lib/utils/k8s/health" client "github.com/devtron-labs/devtron/api/helm-app" openapi "github.com/devtron-labs/devtron/api/helm-app/openapiClient" openapi2 "github.com/devtron-labs/devtron/api/openapi/openapiClient" "github.com/devtron-labs/devtron/client/argocdServer" application2 "github.com/devtron-labs/devtron/client/argocdServer/application" - "github.com/devtron-labs/devtron/internal/constants" - repository3 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app/status" "github.com/devtron-labs/devtron/pkg/appStatus" appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" + repository2 "github.com/devtron-labs/devtron/pkg/appStore/chartGroup/repository" appStoreDeploymentCommon "github.com/devtron-labs/devtron/pkg/appStore/deployment/common" appStoreDeploymentFullMode "github.com/devtron-labs/devtron/pkg/appStore/deployment/fullMode" "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" @@ -31,7 +28,6 @@ import ( "github.com/devtron-labs/devtron/pkg/auth/user" clusterRepository "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/sql" - "github.com/devtron-labs/devtron/util/argo" "github.com/go-pg/pg" "github.com/golang/protobuf/ptypes/timestamp" "go.opentelemetry.io/otel" @@ -44,19 +40,14 @@ import ( type AppStoreDeploymentArgoCdService interface { //InstallApp(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, ctx context.Context) (*appStoreBean.InstallAppVersionDTO, error) InstallApp(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, chartGitAttr *util.ChartGitAttribute, ctx context.Context, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) - GetAppStatus(installedAppAndEnvDetails repository.InstalledAppAndEnvDetails, w http.ResponseWriter, r *http.Request, token string) (string, error) DeleteInstalledApp(ctx context.Context, appName string, environmentName string, installAppVersionRequest *appStoreBean.InstallAppVersionDTO, installedApps *repository.InstalledApps, dbTransaction *pg.Tx) error RollbackRelease(ctx context.Context, installedApp *appStoreBean.InstallAppVersionDTO, deploymentVersion int32, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, bool, error) GetDeploymentHistory(ctx context.Context, installedApp *appStoreBean.InstallAppVersionDTO) (*client.HelmAppDeploymentHistory, error) GetDeploymentHistoryInfo(ctx context.Context, installedApp *appStoreBean.InstallAppVersionDTO, version int32) (*openapi.HelmAppDeploymentManifestDetail, error) GetGitOpsRepoName(appName string, environmentName string) (string, error) - OnUpdateRepoInInstalledApp(ctx context.Context, installAppVersionRequest *appStoreBean.InstallAppVersionDTO, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) - UpdateRequirementDependencies(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, appStoreAppVersion *appStoreDiscoverRepository.AppStoreApplicationVersion) error - UpdateInstalledApp(ctx context.Context, installAppVersionRequest *appStoreBean.InstallAppVersionDTO, environment *clusterRepository.Environment, installedAppVersion *repository.InstalledAppVersions, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) DeleteDeploymentApp(ctx context.Context, appName string, environmentName string, installAppVersionRequest *appStoreBean.InstallAppVersionDTO) error UpdateInstalledAppAndPipelineStatusForFailedDeploymentStatus(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, triggeredAt time.Time, err error) error SaveTimelineForACDHelmApps(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, status string, statusDetail string, statusTime time.Time, tx *pg.Tx) error - UpdateChartInfo(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, ChartGitAttribute *util.ChartGitAttribute, installedAppVersionHistoryId int, ctx context.Context) error UpdateAndSyncACDApps(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, ChartGitAttribute *util.ChartGitAttribute, isMonoRepoMigrationRequired bool, ctx context.Context, tx *pg.Tx) error } @@ -64,31 +55,29 @@ type AppStoreDeploymentArgoCdServiceImpl struct { Logger *zap.SugaredLogger appStoreDeploymentFullModeService appStoreDeploymentFullMode.AppStoreDeploymentFullModeService acdClient application2.ServiceClient - chartGroupDeploymentRepository repository.ChartGroupDeploymentRepository + chartGroupDeploymentRepository repository2.ChartGroupDeploymentRepository installedAppRepository repository.InstalledAppRepository installedAppRepositoryHistory repository.InstalledAppVersionHistoryRepository - chartTemplateService util.ChartTemplateService gitFactory *util.GitFactory - argoUserService argo.ArgoUserService appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService helmAppService client.HelmAppService - gitOpsConfigRepository repository3.GitOpsConfigRepository appStatusService appStatus.AppStatusService pipelineStatusTimelineService status.PipelineStatusTimelineService - userService user.UserService pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository + userService user.UserService appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository argoClientWrapperService argocdServer.ArgoClientWrapperService acdConfig *argocdServer.ACDConfig } func NewAppStoreDeploymentArgoCdServiceImpl(logger *zap.SugaredLogger, appStoreDeploymentFullModeService appStoreDeploymentFullMode.AppStoreDeploymentFullModeService, - acdClient application2.ServiceClient, chartGroupDeploymentRepository repository.ChartGroupDeploymentRepository, - installedAppRepository repository.InstalledAppRepository, installedAppRepositoryHistory repository.InstalledAppVersionHistoryRepository, chartTemplateService util.ChartTemplateService, - gitFactory *util.GitFactory, argoUserService argo.ArgoUserService, appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService, - helmAppService client.HelmAppService, gitOpsConfigRepository repository3.GitOpsConfigRepository, appStatusService appStatus.AppStatusService, - pipelineStatusTimelineService status.PipelineStatusTimelineService, userService user.UserService, + acdClient application2.ServiceClient, chartGroupDeploymentRepository repository2.ChartGroupDeploymentRepository, + installedAppRepository repository.InstalledAppRepository, installedAppRepositoryHistory repository.InstalledAppVersionHistoryRepository, + gitFactory *util.GitFactory, appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService, + helmAppService client.HelmAppService, appStatusService appStatus.AppStatusService, + pipelineStatusTimelineService status.PipelineStatusTimelineService, pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository, + userService user.UserService, appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository, argoClientWrapperService argocdServer.ArgoClientWrapperService, acdConfig *argocdServer.ACDConfig, ) *AppStoreDeploymentArgoCdServiceImpl { @@ -99,16 +88,13 @@ func NewAppStoreDeploymentArgoCdServiceImpl(logger *zap.SugaredLogger, appStoreD chartGroupDeploymentRepository: chartGroupDeploymentRepository, installedAppRepository: installedAppRepository, installedAppRepositoryHistory: installedAppRepositoryHistory, - chartTemplateService: chartTemplateService, gitFactory: gitFactory, - argoUserService: argoUserService, appStoreDeploymentCommonService: appStoreDeploymentCommonService, helmAppService: helmAppService, - gitOpsConfigRepository: gitOpsConfigRepository, appStatusService: appStatusService, pipelineStatusTimelineService: pipelineStatusTimelineService, - userService: userService, pipelineStatusTimelineRepository: pipelineStatusTimelineRepository, + userService: userService, appStoreApplicationVersionRepository: appStoreApplicationVersionRepository, argoClientWrapperService: argoClientWrapperService, acdConfig: acdConfig, @@ -213,55 +199,6 @@ func (impl AppStoreDeploymentArgoCdServiceImpl) InstallApp(installAppVersionRequ // return installAppVersionRequest, nil //} -// TODO: Test ACD to get status -func (impl AppStoreDeploymentArgoCdServiceImpl) GetAppStatus(installedAppAndEnvDetails repository.InstalledAppAndEnvDetails, w http.ResponseWriter, r *http.Request, token string) (string, error) { - if len(installedAppAndEnvDetails.AppName) > 0 && len(installedAppAndEnvDetails.EnvironmentName) > 0 { - acdAppName := installedAppAndEnvDetails.AppName + "-" + installedAppAndEnvDetails.EnvironmentName - query := &application.ResourcesQuery{ - ApplicationName: &acdAppName, - } - ctx, cancel := context.WithCancel(r.Context()) - if cn, ok := w.(http.CloseNotifier); ok { - go func(done <-chan struct{}, closed <-chan bool) { - select { - case <-done: - case <-closed: - cancel() - } - }(ctx.Done(), cn.CloseNotify()) - } - acdToken, err := impl.argoUserService.GetLatestDevtronArgoCdUserToken() - if err != nil { - impl.Logger.Errorw("error in getting acd token", "err", err) - return "", err - } - ctx = context.WithValue(ctx, "token", acdToken) - defer cancel() - impl.Logger.Debugf("Getting status for app %s in env %s", installedAppAndEnvDetails.AppName, installedAppAndEnvDetails.EnvironmentName) - start := time.Now() - resp, err := impl.acdClient.ResourceTree(ctx, query) - elapsed := time.Since(start) - impl.Logger.Debugf("Time elapsed %s in fetching application %s for environment %s", elapsed, installedAppAndEnvDetails.AppName, installedAppAndEnvDetails.EnvironmentName) - if err != nil { - impl.Logger.Errorw("error fetching resource tree", "error", err) - err = &util.ApiError{ - Code: constants.AppDetailResourceTreeNotFound, - InternalMessage: "app detail fetched, failed to get resource tree from acd", - UserMessage: "app detail fetched, failed to get resource tree from acd", - } - return "", err - - } - //use this resp.Status to update app_status table - err = impl.appStatusService.UpdateStatusWithAppIdEnvId(installedAppAndEnvDetails.AppId, installedAppAndEnvDetails.EnvironmentId, resp.Status) - if err != nil { - impl.Logger.Warnw("error in updating app status", "err", err, installedAppAndEnvDetails.AppId, "envId", installedAppAndEnvDetails.EnvironmentId) - } - return resp.Status, nil - } - return "", errors.New("invalid app name or env name") -} - func (impl AppStoreDeploymentArgoCdServiceImpl) DeleteInstalledApp(ctx context.Context, appName string, environmentName string, installAppVersionRequest *appStoreBean.InstallAppVersionDTO, installedApps *repository.InstalledApps, dbTransaction *pg.Tx) error { err := impl.appStatusService.DeleteWithAppIdEnvId(dbTransaction, installedApps.AppId, installedApps.EnvironmentId) @@ -560,104 +497,6 @@ func (impl AppStoreDeploymentArgoCdServiceImpl) GetGitOpsRepoName(appName string return impl.appStoreDeploymentFullModeService.GetGitOpsRepoName(appName, environmentName) } -func (impl *AppStoreDeploymentArgoCdServiceImpl) OnUpdateRepoInInstalledApp(ctx context.Context, installAppVersionRequest *appStoreBean.InstallAppVersionDTO, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) { - //creating deployment started status timeline - timeline := &pipelineConfig.PipelineStatusTimeline{ - InstalledAppVersionHistoryId: installAppVersionRequest.InstalledAppVersionHistoryId, - Status: pipelineConfig.TIMELINE_STATUS_DEPLOYMENT_INITIATED, - StatusDetail: "Deployment initiated successfully.", - StatusTime: time.Now(), - AuditLog: sql.AuditLog{ - CreatedBy: installAppVersionRequest.UserId, - CreatedOn: time.Now(), - UpdatedBy: installAppVersionRequest.UserId, - UpdatedOn: time.Now(), - }, - } - err := impl.pipelineStatusTimelineService.SaveTimeline(timeline, tx, true) - if err != nil { - impl.Logger.Errorw("error in creating timeline status for deployment initiation for update of installedAppVersionHistoryId", "err", err, "installedAppVersionHistoryId", installAppVersionRequest.InstalledAppVersionHistoryId) - } - //git operation pull push - appStoreGitOpsResponse, err := impl.appStoreDeploymentCommonService.GenerateManifestAndPerformGitOperations(installAppVersionRequest) - if err != nil { - impl.Logger.Errorw("error in doing gitops operation", "err", err) - _ = impl.SaveTimelineForACDHelmApps(installAppVersionRequest, pipelineConfig.TIMELINE_STATUS_GIT_COMMIT_FAILED, fmt.Sprintf("Git commit failed - %v", err), time.Now(), tx) - - } - - _ = impl.SaveTimelineForACDHelmApps(installAppVersionRequest, pipelineConfig.TIMELINE_STATUS_GIT_COMMIT, "Git commit done successfully.", time.Now(), tx) - - //acd operation register, sync - installAppVersionRequest, err = impl.patchAcdApp(ctx, installAppVersionRequest, appStoreGitOpsResponse.ChartGitAttribute) - if err != nil { - return installAppVersionRequest, err - } - - return installAppVersionRequest, nil -} - -func (impl *AppStoreDeploymentArgoCdServiceImpl) UpdateRequirementDependencies(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, appStoreAppVersion *appStoreDiscoverRepository.AppStoreApplicationVersion) error { - RequirementsString, err := impl.appStoreDeploymentCommonService.GetRequirementsString(installAppVersionRequest.AppStoreVersion) - if err != nil { - impl.Logger.Errorw("error in building requirements config for helm app", "err", err) - return err - } - requirementsGitConfig, err := impl.appStoreDeploymentCommonService.GetGitCommitConfig(installAppVersionRequest, RequirementsString, appStoreBean.REQUIREMENTS_YAML_FILE) - if err != nil { - impl.Logger.Errorw("error in getting git config for helm app", "err", err) - return err - } - _, err = impl.appStoreDeploymentCommonService.CommitConfigToGit(requirementsGitConfig) - if err != nil { - impl.Logger.Errorw("error in committing config to git for helm app", "err", err) - return err - } - return nil -} - -func (impl AppStoreDeploymentArgoCdServiceImpl) UpdateInstalledApp(ctx context.Context, installAppVersionRequest *appStoreBean.InstallAppVersionDTO, environment *clusterRepository.Environment, installedAppVersion *repository.InstalledAppVersions, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) { - //creating deployment started status timeline when mono repo migration is not required - timeline := &pipelineConfig.PipelineStatusTimeline{ - InstalledAppVersionHistoryId: installAppVersionRequest.InstalledAppVersionHistoryId, - Status: pipelineConfig.TIMELINE_STATUS_DEPLOYMENT_INITIATED, - StatusDetail: "Deployment initiated successfully.", - StatusTime: time.Now(), - AuditLog: sql.AuditLog{ - CreatedBy: installAppVersionRequest.UserId, - CreatedOn: time.Now(), - UpdatedBy: installAppVersionRequest.UserId, - UpdatedOn: time.Now(), - }, - } - err := impl.pipelineStatusTimelineService.SaveTimeline(timeline, tx, true) - if err != nil { - impl.Logger.Errorw("error in creating timeline status for deployment initiation for update of installedAppVersionHistoryId", "err", err, "installedAppVersionHistoryId", installAppVersionRequest.InstalledAppVersionHistoryId) - } - //update values yaml in chart - installAppVersionRequest, err = impl.updateValuesYaml(environment, installedAppVersion, installAppVersionRequest, tx) - if err != nil { - impl.Logger.Errorw("error while commit values to git", "error", err) - noTargetFound, _ := impl.appStoreDeploymentCommonService.ParseGitRepoErrorResponse(err) - if noTargetFound { - //if by mistake no content found while updating git repo, do auto fix - installAppVersionRequest, err = impl.OnUpdateRepoInInstalledApp(ctx, installAppVersionRequest, tx) - if err != nil { - impl.Logger.Errorw("error while update repo on helm update", "error", err) - return nil, err - } - } else { - return nil, err - } - } - installAppVersionRequest.Environment = environment - - //ACD sync operation - //impl.appStoreDeploymentFullModeService.SyncACD(installAppVersionRequest.ACDAppName, ctx) - - return installAppVersionRequest, nil -} - func (impl AppStoreDeploymentArgoCdServiceImpl) patchAcdApp(ctx context.Context, installAppVersionRequest *appStoreBean.InstallAppVersionDTO, chartGitAttr *util.ChartGitAttribute) (*appStoreBean.InstallAppVersionDTO, error) { ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) defer cancel() @@ -740,115 +579,3 @@ func (impl AppStoreDeploymentArgoCdServiceImpl) DeleteDeploymentApp(ctx context. } return nil } - -func (impl AppStoreDeploymentArgoCdServiceImpl) UpdateInstalledAppAndPipelineStatusForFailedDeploymentStatus(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, triggeredAt time.Time, err error) error { - if err != nil { - terminalStatusExists, timelineErr := impl.pipelineStatusTimelineRepository.CheckIfTerminalStatusTimelinePresentByInstalledAppVersionHistoryId(installAppVersionRequest.InstalledAppVersionHistoryId) - if timelineErr != nil { - impl.Logger.Errorw("error in checking if terminal status timeline exists by installedAppVersionHistoryId", "err", timelineErr, "installedAppVersionHistoryId", installAppVersionRequest.InstalledAppVersionHistoryId) - return timelineErr - } - if !terminalStatusExists { - impl.Logger.Infow("marking pipeline deployment failed", "err", err) - timeline := &pipelineConfig.PipelineStatusTimeline{ - InstalledAppVersionHistoryId: installAppVersionRequest.InstalledAppVersionHistoryId, - Status: pipelineConfig.TIMELINE_STATUS_DEPLOYMENT_FAILED, - StatusDetail: fmt.Sprintf("Deployment failed: %v", err), - StatusTime: time.Now(), - AuditLog: sql.AuditLog{ - CreatedBy: 1, - CreatedOn: time.Now(), - UpdatedBy: 1, - UpdatedOn: time.Now(), - }, - } - isAppStore := true - timelineErr = impl.pipelineStatusTimelineService.SaveTimeline(timeline, nil, isAppStore) - if timelineErr != nil { - impl.Logger.Errorw("error in creating timeline status for deployment fail", "err", timelineErr, "timeline", timeline) - } - } - impl.Logger.Errorw("error in triggering installed application deployment, setting status as fail ", "versionHistoryId", installAppVersionRequest.InstalledAppVersionHistoryId, "err", err) - - installedAppVersionHistory, err := impl.installedAppRepositoryHistory.GetInstalledAppVersionHistory(installAppVersionRequest.InstalledAppVersionHistoryId) - if err != nil { - impl.Logger.Errorw("error in getting installedAppVersionHistory by installedAppVersionHistoryId", "installedAppVersionHistoryId", installAppVersionRequest.InstalledAppVersionHistoryId, "err", err) - return err - } - installedAppVersionHistory.Status = pipelineConfig.WorkflowFailed - installedAppVersionHistory.FinishedOn = triggeredAt - installedAppVersionHistory.UpdatedOn = time.Now() - installedAppVersionHistory.UpdatedBy = installAppVersionRequest.UserId - _, err = impl.installedAppRepositoryHistory.UpdateInstalledAppVersionHistory(installedAppVersionHistory, nil) - if err != nil { - impl.Logger.Errorw("error updating installed app version history status", "err", err, "installedAppVersionHistory", installedAppVersionHistory) - return err - } - - } else { - //update [n,n-1] statuses as failed if not terminal - terminalStatus := []string{string(health.HealthStatusHealthy), pipelineConfig.WorkflowAborted, pipelineConfig.WorkflowFailed, pipelineConfig.WorkflowSucceeded} - previousNonTerminalHistory, err := impl.installedAppRepositoryHistory.FindPreviousInstalledAppVersionHistoryByStatus(installAppVersionRequest.Id, installAppVersionRequest.InstalledAppVersionHistoryId, terminalStatus) - if err != nil { - impl.Logger.Errorw("error fetching previous installed app version history, updating installed app version history status,", "err", err, "installAppVersionRequest", installAppVersionRequest) - return err - } else if len(previousNonTerminalHistory) == 0 { - impl.Logger.Errorw("no previous history found in updating installedAppVersionHistory status,", "err", err, "installAppVersionRequest", installAppVersionRequest) - return nil - } - dbConnection := impl.installedAppRepositoryHistory.GetConnection() - tx, err := dbConnection.Begin() - if err != nil { - impl.Logger.Errorw("error on update status, txn begin failed", "err", err) - return err - } - // Rollback tx on error. - defer tx.Rollback() - var timelines []*pipelineConfig.PipelineStatusTimeline - for _, previousHistory := range previousNonTerminalHistory { - if previousHistory.Status == string(health.HealthStatusHealthy) || - previousHistory.Status == pipelineConfig.WorkflowSucceeded || - previousHistory.Status == pipelineConfig.WorkflowAborted || - previousHistory.Status == pipelineConfig.WorkflowFailed { - //terminal status return - impl.Logger.Infow("skip updating installedAppVersionHistory status as previous history status is", "status", previousHistory.Status) - continue - } - impl.Logger.Infow("updating installedAppVersionHistory status as previous runner status is", "status", previousHistory.Status) - previousHistory.FinishedOn = triggeredAt - previousHistory.Status = pipelineConfig.WorkflowFailed - previousHistory.UpdatedOn = time.Now() - previousHistory.UpdatedBy = installAppVersionRequest.UserId - timeline := &pipelineConfig.PipelineStatusTimeline{ - InstalledAppVersionHistoryId: previousHistory.Id, - Status: pipelineConfig.TIMELINE_STATUS_DEPLOYMENT_SUPERSEDED, - StatusDetail: "This deployment is superseded.", - StatusTime: time.Now(), - AuditLog: sql.AuditLog{ - CreatedBy: 1, - CreatedOn: time.Now(), - UpdatedBy: 1, - UpdatedOn: time.Now(), - }, - } - timelines = append(timelines, timeline) - } - - err = impl.installedAppRepositoryHistory.UpdateInstalledAppVersionHistoryWithTxn(previousNonTerminalHistory, tx) - if err != nil { - impl.Logger.Errorw("error updating cd wf runner status", "err", err, "previousNonTerminalHistory", previousNonTerminalHistory) - return err - } - err = impl.pipelineStatusTimelineRepository.SaveTimelinesWithTxn(timelines, tx) - if err != nil { - impl.Logger.Errorw("error updating pipeline status timelines", "err", err, "timelines", timelines) - return err - } - err = tx.Commit() - if err != nil { - impl.Logger.Errorw("error in db transaction commit", "err", err) - return err - } - } - return nil -} diff --git a/pkg/appStore/deployment/tool/AppStoreDeploymentHelmService.go b/pkg/appStore/deployment/tool/AppStoreDeploymentHelmService.go index 9716abb607..fd94dbad26 100644 --- a/pkg/appStore/deployment/tool/AppStoreDeploymentHelmService.go +++ b/pkg/appStore/deployment/tool/AppStoreDeploymentHelmService.go @@ -4,18 +4,15 @@ import ( "context" "errors" repository2 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" - "net/http" "time" client "github.com/devtron-labs/devtron/api/helm-app" openapi "github.com/devtron-labs/devtron/api/helm-app/openapiClient" - "github.com/devtron-labs/devtron/internal/constants" "github.com/devtron-labs/devtron/internal/util" appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" appStoreDeploymentCommon "github.com/devtron-labs/devtron/pkg/appStore/deployment/common" "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" appStoreDiscoverRepository "github.com/devtron-labs/devtron/pkg/appStore/discover/repository" - clusterRepository "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/go-pg/pg" "go.opentelemetry.io/otel" "go.uber.org/zap" @@ -24,18 +21,11 @@ import ( type AppStoreDeploymentHelmService interface { InstallApp(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, chartGitAttr *util.ChartGitAttribute, ctx context.Context, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) - GetAppStatus(installedAppAndEnvDetails repository.InstalledAppAndEnvDetails, w http.ResponseWriter, r *http.Request, token string) (string, error) DeleteInstalledApp(ctx context.Context, appName string, environmentName string, installAppVersionRequest *appStoreBean.InstallAppVersionDTO, installedApps *repository.InstalledApps, dbTransaction *pg.Tx) error RollbackRelease(ctx context.Context, installedApp *appStoreBean.InstallAppVersionDTO, deploymentVersion int32, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, bool, error) GetDeploymentHistory(ctx context.Context, installedApp *appStoreBean.InstallAppVersionDTO) (*client.HelmAppDeploymentHistory, error) GetDeploymentHistoryInfo(ctx context.Context, installedApp *appStoreBean.InstallAppVersionDTO, version int32) (*openapi.HelmAppDeploymentManifestDetail, error) - GetGitOpsRepoName(appName string, environmentName string) (string, error) - OnUpdateRepoInInstalledApp(ctx context.Context, installAppVersionRequest *appStoreBean.InstallAppVersionDTO, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) - UpdateRequirementDependencies(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, appStoreAppVersion *appStoreDiscoverRepository.AppStoreApplicationVersion) error - UpdateInstalledApp(ctx context.Context, installAppVersionRequest *appStoreBean.InstallAppVersionDTO, environment *clusterRepository.Environment, installedAppVersion *repository.InstalledAppVersions, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) DeleteDeploymentApp(ctx context.Context, appName string, environmentName string, installAppVersionRequest *appStoreBean.InstallAppVersionDTO) error - UpdateInstalledAppAndPipelineStatusForFailedDeploymentStatus(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, triggeredAt time.Time, err error) error - SaveTimelineForACDHelmApps(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, status string, statusDetail string, statusTime time.Time, tx *pg.Tx) error UpdateChartInfo(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, ChartGitAttribute *util.ChartGitAttribute, installedAppVersionHistoryId int, ctx context.Context) error } @@ -43,7 +33,6 @@ type AppStoreDeploymentHelmServiceImpl struct { Logger *zap.SugaredLogger helmAppService client.HelmAppService appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository - environmentRepository clusterRepository.EnvironmentRepository helmAppClient client.HelmAppClient installedAppRepository repository.InstalledAppRepository appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService @@ -51,12 +40,11 @@ type AppStoreDeploymentHelmServiceImpl struct { } func NewAppStoreDeploymentHelmServiceImpl(logger *zap.SugaredLogger, helmAppService client.HelmAppService, appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository, - environmentRepository clusterRepository.EnvironmentRepository, helmAppClient client.HelmAppClient, installedAppRepository repository.InstalledAppRepository, appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService, OCIRegistryConfigRepository repository2.OCIRegistryConfigRepository) *AppStoreDeploymentHelmServiceImpl { + helmAppClient client.HelmAppClient, installedAppRepository repository.InstalledAppRepository, appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService, OCIRegistryConfigRepository repository2.OCIRegistryConfigRepository) *AppStoreDeploymentHelmServiceImpl { return &AppStoreDeploymentHelmServiceImpl{ Logger: logger, helmAppService: helmAppService, appStoreApplicationVersionRepository: appStoreApplicationVersionRepository, - environmentRepository: environmentRepository, helmAppClient: helmAppClient, installedAppRepository: installedAppRepository, appStoreDeploymentCommonService: appStoreDeploymentCommonService, @@ -138,38 +126,6 @@ func (impl AppStoreDeploymentHelmServiceImpl) InstallApp(installAppVersionReques return installAppVersionRequest, nil } -func (impl AppStoreDeploymentHelmServiceImpl) GetAppStatus(installedAppAndEnvDetails repository.InstalledAppAndEnvDetails, w http.ResponseWriter, r *http.Request, token string) (string, error) { - - environment, err := impl.environmentRepository.FindById(installedAppAndEnvDetails.EnvironmentId) - if err != nil { - impl.Logger.Errorw("Error in getting environment", "err", err) - return "", err - } - - appIdentifier := &client.AppIdentifier{ - ClusterId: environment.ClusterId, - Namespace: environment.Namespace, - ReleaseName: installedAppAndEnvDetails.AppName, - } - - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - defer cancel() - - appDetail, err := impl.helmAppService.GetApplicationDetail(ctx, appIdentifier) - if err != nil { - // handling like argocd - impl.Logger.Errorw("error fetching helm app resource tree", "error", err, "appIdentifier", appIdentifier) - err = &util.ApiError{ - Code: constants.AppDetailResourceTreeNotFound, - InternalMessage: "Failed to get resource tree from helm", - UserMessage: "Failed to get resource tree from helm", - } - return "", err - } - - return appDetail.ApplicationStatus, nil -} - func (impl AppStoreDeploymentHelmServiceImpl) DeleteInstalledApp(ctx context.Context, appName string, environmentName string, installAppVersionRequest *appStoreBean.InstallAppVersionDTO, installedApps *repository.InstalledApps, dbTransaction *pg.Tx) error { if installAppVersionRequest.ForceDelete { return nil @@ -294,42 +250,6 @@ func (impl *AppStoreDeploymentHelmServiceImpl) GetGitOpsRepoName(appName string, return "", errors.New("method GetGitOpsRepoName not implemented") } -func (impl *AppStoreDeploymentHelmServiceImpl) OnUpdateRepoInInstalledApp(ctx context.Context, installAppVersionRequest *appStoreBean.InstallAppVersionDTO, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) { - //TODO: gitOps operations here based on flag - if installAppVersionRequest.PerformGitOpsForHelmApp { - _, err := impl.appStoreDeploymentCommonService.GenerateManifestAndPerformGitOperations(installAppVersionRequest) - if err != nil { - return installAppVersionRequest, err - } - } - - err := impl.updateApplicationWithChartInfo(ctx, installAppVersionRequest.InstalledAppId, installAppVersionRequest.AppStoreVersion, installAppVersionRequest.ValuesOverrideYaml, installAppVersionRequest.InstalledAppVersionHistoryId) - if err != nil { - return installAppVersionRequest, err - } - return installAppVersionRequest, err -} - -func (impl *AppStoreDeploymentHelmServiceImpl) UpdateRequirementDependencies(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, appStoreAppVersion *appStoreDiscoverRepository.AppStoreApplicationVersion) error { - RequirementsString, err := impl.appStoreDeploymentCommonService.GetRequirementsString(installAppVersionRequest.AppStoreVersion) - if err != nil { - impl.Logger.Errorw("error in building requirements config for helm app", "err", err) - return err - } - requirementsGitConfig, err := impl.appStoreDeploymentCommonService.GetGitCommitConfig(installAppVersionRequest, RequirementsString, appStoreBean.REQUIREMENTS_YAML_FILE) - if err != nil { - impl.Logger.Errorw("error in getting git config for helm app", "err", err) - return err - } - _, err = impl.appStoreDeploymentCommonService.CommitConfigToGit(requirementsGitConfig) - if err != nil { - impl.Logger.Errorw("error in committing config to git for helm app", "err", err) - return err - } - return nil - //return errors.New("method UpdateRequirementDependencies not implemented") -} - func (impl *AppStoreDeploymentHelmServiceImpl) UpdateValuesDependencies(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) error { appStoreAppVersion, err := impl.appStoreApplicationVersionRepository.FindById(installAppVersionRequest.AppStoreVersion) if err != nil { @@ -354,37 +274,6 @@ func (impl *AppStoreDeploymentHelmServiceImpl) UpdateValuesDependencies(installA return nil } -func (impl *AppStoreDeploymentHelmServiceImpl) UpdateInstalledApp(ctx context.Context, installAppVersionRequest *appStoreBean.InstallAppVersionDTO, environment *clusterRepository.Environment, installedAppVersion *repository.InstalledAppVersions, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) { - - noTargetFound := false - - if installAppVersionRequest.PerformGitOps { - err := impl.UpdateValuesDependencies(installAppVersionRequest) - if err != nil { - impl.Logger.Errorw("error while commit values to git", "error", err) - noTargetFound, _ = impl.appStoreDeploymentCommonService.ParseGitRepoErrorResponse(err) - if noTargetFound { - //if by mistake no content found while updating git repo, do auto fix - installAppVersionRequest, err = impl.OnUpdateRepoInInstalledApp(ctx, installAppVersionRequest, tx) - if err != nil { - impl.Logger.Errorw("error while update repo on helm update", "error", err) - return nil, err - } - } else { - return nil, err - } - } - } - if !noTargetFound { - // update chart application already called, hence skipping - err := impl.updateApplicationWithChartInfo(ctx, installAppVersionRequest.InstalledAppId, installAppVersionRequest.AppStoreVersion, installAppVersionRequest.ValuesOverrideYaml, 0) - if err != nil { - return nil, err - } - } - return installAppVersionRequest, nil -} - func (impl *AppStoreDeploymentHelmServiceImpl) updateApplicationWithChartInfo(ctx context.Context, installedAppId int, appStoreApplicationVersionId int, valuesOverrideYaml string, installAppVersionHistoryId int) error { installedApp, err := impl.installedAppRepository.GetInstalledApp(installedAppId) diff --git a/pkg/appStore/deployment/tool/DeploymentStatusService.go b/pkg/appStore/deployment/tool/DeploymentStatusService.go new file mode 100644 index 0000000000..fe2de9c816 --- /dev/null +++ b/pkg/appStore/deployment/tool/DeploymentStatusService.go @@ -0,0 +1,122 @@ +package appStoreDeploymentTool + +import ( + "fmt" + "github.com/devtron-labs/common-lib/utils/k8s/health" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/pkg/appStore/bean" + "github.com/devtron-labs/devtron/pkg/sql" + "time" +) + +func (impl AppStoreDeploymentArgoCdServiceImpl) UpdateInstalledAppAndPipelineStatusForFailedDeploymentStatus(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, triggeredAt time.Time, err error) error { + if err != nil { + terminalStatusExists, timelineErr := impl.pipelineStatusTimelineRepository.CheckIfTerminalStatusTimelinePresentByInstalledAppVersionHistoryId(installAppVersionRequest.InstalledAppVersionHistoryId) + if timelineErr != nil { + impl.Logger.Errorw("error in checking if terminal status timeline exists by installedAppVersionHistoryId", "err", timelineErr, "installedAppVersionHistoryId", installAppVersionRequest.InstalledAppVersionHistoryId) + return timelineErr + } + if !terminalStatusExists { + impl.Logger.Infow("marking pipeline deployment failed", "err", err) + timeline := &pipelineConfig.PipelineStatusTimeline{ + InstalledAppVersionHistoryId: installAppVersionRequest.InstalledAppVersionHistoryId, + Status: pipelineConfig.TIMELINE_STATUS_DEPLOYMENT_FAILED, + StatusDetail: fmt.Sprintf("Deployment failed: %v", err), + StatusTime: time.Now(), + AuditLog: sql.AuditLog{ + CreatedBy: 1, + CreatedOn: time.Now(), + UpdatedBy: 1, + UpdatedOn: time.Now(), + }, + } + isAppStore := true + timelineErr = impl.pipelineStatusTimelineService.SaveTimeline(timeline, nil, isAppStore) + if timelineErr != nil { + impl.Logger.Errorw("error in creating timeline status for deployment fail", "err", timelineErr, "timeline", timeline) + } + } + impl.Logger.Errorw("error in triggering installed application deployment, setting status as fail ", "versionHistoryId", installAppVersionRequest.InstalledAppVersionHistoryId, "err", err) + + installedAppVersionHistory, err := impl.installedAppRepositoryHistory.GetInstalledAppVersionHistory(installAppVersionRequest.InstalledAppVersionHistoryId) + if err != nil { + impl.Logger.Errorw("error in getting installedAppVersionHistory by installedAppVersionHistoryId", "installedAppVersionHistoryId", installAppVersionRequest.InstalledAppVersionHistoryId, "err", err) + return err + } + installedAppVersionHistory.Status = pipelineConfig.WorkflowFailed + installedAppVersionHistory.FinishedOn = triggeredAt + installedAppVersionHistory.UpdatedOn = time.Now() + installedAppVersionHistory.UpdatedBy = installAppVersionRequest.UserId + _, err = impl.installedAppRepositoryHistory.UpdateInstalledAppVersionHistory(installedAppVersionHistory, nil) + if err != nil { + impl.Logger.Errorw("error updating installed app version history status", "err", err, "installedAppVersionHistory", installedAppVersionHistory) + return err + } + + } else { + //update [n,n-1] statuses as failed if not terminal + terminalStatus := []string{string(health.HealthStatusHealthy), pipelineConfig.WorkflowAborted, pipelineConfig.WorkflowFailed, pipelineConfig.WorkflowSucceeded} + previousNonTerminalHistory, err := impl.installedAppRepositoryHistory.FindPreviousInstalledAppVersionHistoryByStatus(installAppVersionRequest.Id, installAppVersionRequest.InstalledAppVersionHistoryId, terminalStatus) + if err != nil { + impl.Logger.Errorw("error fetching previous installed app version history, updating installed app version history status,", "err", err, "installAppVersionRequest", installAppVersionRequest) + return err + } else if len(previousNonTerminalHistory) == 0 { + impl.Logger.Errorw("no previous history found in updating installedAppVersionHistory status,", "err", err, "installAppVersionRequest", installAppVersionRequest) + return nil + } + dbConnection := impl.installedAppRepositoryHistory.GetConnection() + tx, err := dbConnection.Begin() + if err != nil { + impl.Logger.Errorw("error on update status, txn begin failed", "err", err) + return err + } + // Rollback tx on error. + defer tx.Rollback() + var timelines []*pipelineConfig.PipelineStatusTimeline + for _, previousHistory := range previousNonTerminalHistory { + if previousHistory.Status == string(health.HealthStatusHealthy) || + previousHistory.Status == pipelineConfig.WorkflowSucceeded || + previousHistory.Status == pipelineConfig.WorkflowAborted || + previousHistory.Status == pipelineConfig.WorkflowFailed { + //terminal status return + impl.Logger.Infow("skip updating installedAppVersionHistory status as previous history status is", "status", previousHistory.Status) + continue + } + impl.Logger.Infow("updating installedAppVersionHistory status as previous runner status is", "status", previousHistory.Status) + previousHistory.FinishedOn = triggeredAt + previousHistory.Status = pipelineConfig.WorkflowFailed + previousHistory.UpdatedOn = time.Now() + previousHistory.UpdatedBy = installAppVersionRequest.UserId + timeline := &pipelineConfig.PipelineStatusTimeline{ + InstalledAppVersionHistoryId: previousHistory.Id, + Status: pipelineConfig.TIMELINE_STATUS_DEPLOYMENT_SUPERSEDED, + StatusDetail: "This deployment is superseded.", + StatusTime: time.Now(), + AuditLog: sql.AuditLog{ + CreatedBy: 1, + CreatedOn: time.Now(), + UpdatedBy: 1, + UpdatedOn: time.Now(), + }, + } + timelines = append(timelines, timeline) + } + + err = impl.installedAppRepositoryHistory.UpdateInstalledAppVersionHistoryWithTxn(previousNonTerminalHistory, tx) + if err != nil { + impl.Logger.Errorw("error updating cd wf runner status", "err", err, "previousNonTerminalHistory", previousNonTerminalHistory) + return err + } + err = impl.pipelineStatusTimelineRepository.SaveTimelinesWithTxn(timelines, tx) + if err != nil { + impl.Logger.Errorw("error updating pipeline status timelines", "err", err, "timelines", timelines) + return err + } + err = tx.Commit() + if err != nil { + impl.Logger.Errorw("error in db transaction commit", "err", err) + return err + } + } + return nil +} diff --git a/scripts/sql/210_unused_integration.up.sql b/scripts/sql/211_unused_integration.up.sql similarity index 100% rename from scripts/sql/210_unused_integration.up.sql rename to scripts/sql/211_unused_integration.up.sql diff --git a/scripts/sql/210_unused_intergration.down.sql b/scripts/sql/211_unused_intergration.down.sql similarity index 100% rename from scripts/sql/210_unused_intergration.down.sql rename to scripts/sql/211_unused_intergration.down.sql diff --git a/wire_gen.go b/wire_gen.go index e871688b17..a0bd21ec8a 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -61,7 +61,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" repository5 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" "github.com/devtron-labs/devtron/internal/sql/repository/helper" - repository13 "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging" + repository14 "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/sql/repository/resourceGroup" "github.com/devtron-labs/devtron/internal/sql/repository/security" @@ -74,13 +74,14 @@ import ( "github.com/devtron-labs/devtron/pkg/appClone/batch" appStatus2 "github.com/devtron-labs/devtron/pkg/appStatus" "github.com/devtron-labs/devtron/pkg/appStore/bean" + "github.com/devtron-labs/devtron/pkg/appStore/chartGroup" + repository11 "github.com/devtron-labs/devtron/pkg/appStore/chartGroup/repository" "github.com/devtron-labs/devtron/pkg/appStore/chartProvider" "github.com/devtron-labs/devtron/pkg/appStore/deployment/common" "github.com/devtron-labs/devtron/pkg/appStore/deployment/fullMode" repository3 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" "github.com/devtron-labs/devtron/pkg/appStore/deployment/service" "github.com/devtron-labs/devtron/pkg/appStore/deployment/tool" - "github.com/devtron-labs/devtron/pkg/appStore/deployment/tool/gitops" "github.com/devtron-labs/devtron/pkg/appStore/discover/repository" service3 "github.com/devtron-labs/devtron/pkg/appStore/discover/service" "github.com/devtron-labs/devtron/pkg/appStore/values/repository" @@ -116,7 +117,7 @@ import ( "github.com/devtron-labs/devtron/pkg/k8s/capacity" "github.com/devtron-labs/devtron/pkg/k8s/informer" "github.com/devtron-labs/devtron/pkg/kubernetesResourceAuditLogs" - repository14 "github.com/devtron-labs/devtron/pkg/kubernetesResourceAuditLogs/repository" + repository15 "github.com/devtron-labs/devtron/pkg/kubernetesResourceAuditLogs/repository" "github.com/devtron-labs/devtron/pkg/module" "github.com/devtron-labs/devtron/pkg/module/repo" "github.com/devtron-labs/devtron/pkg/module/store" @@ -125,10 +126,10 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/history" repository6 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" - repository11 "github.com/devtron-labs/devtron/pkg/pipeline/repository" + repository12 "github.com/devtron-labs/devtron/pkg/pipeline/repository" "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/plugin" - repository12 "github.com/devtron-labs/devtron/pkg/plugin/repository" + repository13 "github.com/devtron-labs/devtron/pkg/plugin/repository" resourceGroup2 "github.com/devtron-labs/devtron/pkg/resourceGroup" "github.com/devtron-labs/devtron/pkg/resourceQualifiers" security2 "github.com/devtron-labs/devtron/pkg/security" @@ -401,26 +402,26 @@ func InitializeApp() (*App, error) { return nil, err } appStatusServiceImpl := appStatus2.NewAppStatusServiceImpl(appStatusRepositoryImpl, sugaredLogger, enforcerImpl, enforcerUtilImpl) - chartGroupDeploymentRepositoryImpl := repository3.NewChartGroupDeploymentRepositoryImpl(db, sugaredLogger) + chartGroupDeploymentRepositoryImpl := repository11.NewChartGroupDeploymentRepositoryImpl(db, sugaredLogger) clusterInstalledAppsRepositoryImpl := repository3.NewClusterInstalledAppsRepositoryImpl(db, sugaredLogger) refChartProxyDir := _wireRefChartProxyDirValue appStoreDeploymentCommonServiceImpl := appStoreDeploymentCommon.NewAppStoreDeploymentCommonServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, chartTemplateServiceImpl, refChartProxyDir, gitFactory, gitOpsConfigRepositoryImpl) ociRegistryConfigRepositoryImpl := repository5.NewOCIRegistryConfigRepositoryImpl(db) - appStoreDeploymentHelmServiceImpl := appStoreDeploymentTool.NewAppStoreDeploymentHelmServiceImpl(sugaredLogger, helmAppServiceImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, helmAppClientImpl, installedAppRepositoryImpl, appStoreDeploymentCommonServiceImpl, ociRegistryConfigRepositoryImpl) + appStoreDeploymentHelmServiceImpl := appStoreDeploymentTool.NewAppStoreDeploymentHelmServiceImpl(sugaredLogger, helmAppServiceImpl, appStoreApplicationVersionRepositoryImpl, helmAppClientImpl, installedAppRepositoryImpl, appStoreDeploymentCommonServiceImpl, ociRegistryConfigRepositoryImpl) acdConfig, err := argocdServer.GetACDDeploymentConfig() if err != nil { return nil, err } argoClientWrapperServiceImpl := argocdServer.NewArgoClientWrapperServiceImpl(sugaredLogger, applicationServiceClientImpl, acdConfig) - appStoreDeploymentFullModeServiceImpl := appStoreDeploymentFullMode.NewAppStoreDeploymentFullModeServiceImpl(sugaredLogger, chartTemplateServiceImpl, refChartProxyDir, repositoryServiceClientImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, applicationServiceClientImpl, argoK8sClientImpl, gitFactory, acdAuthConfig, globalEnvVariables, installedAppRepositoryImpl, tokenCache, argoUserServiceImpl, gitOpsConfigRepositoryImpl, pipelineStatusTimelineServiceImpl, appStoreDeploymentCommonServiceImpl, argoClientWrapperServiceImpl, pubSubClientServiceImpl, installedAppVersionHistoryRepositoryImpl, acdConfig) - appStoreDeploymentArgoCdServiceImpl := appStoreDeploymentGitopsTool.NewAppStoreDeploymentArgoCdServiceImpl(sugaredLogger, appStoreDeploymentFullModeServiceImpl, applicationServiceClientImpl, chartGroupDeploymentRepositoryImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, chartTemplateServiceImpl, gitFactory, argoUserServiceImpl, appStoreDeploymentCommonServiceImpl, helmAppServiceImpl, gitOpsConfigRepositoryImpl, appStatusServiceImpl, pipelineStatusTimelineServiceImpl, userServiceImpl, pipelineStatusTimelineRepositoryImpl, appStoreApplicationVersionRepositoryImpl, argoClientWrapperServiceImpl, acdConfig) + appStoreDeploymentFullModeServiceImpl := appStoreDeploymentFullMode.NewAppStoreDeploymentFullModeServiceImpl(sugaredLogger, chartTemplateServiceImpl, repositoryServiceClientImpl, applicationServiceClientImpl, argoK8sClientImpl, acdAuthConfig, argoUserServiceImpl, pipelineStatusTimelineServiceImpl, appStoreDeploymentCommonServiceImpl, argoClientWrapperServiceImpl, pubSubClientServiceImpl, installedAppVersionHistoryRepositoryImpl, acdConfig) + appStoreDeploymentArgoCdServiceImpl := appStoreDeploymentTool.NewAppStoreDeploymentArgoCdServiceImpl(sugaredLogger, appStoreDeploymentFullModeServiceImpl, applicationServiceClientImpl, chartGroupDeploymentRepositoryImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, gitFactory, appStoreDeploymentCommonServiceImpl, helmAppServiceImpl, appStatusServiceImpl, pipelineStatusTimelineServiceImpl, pipelineStatusTimelineRepositoryImpl, userServiceImpl, appStoreApplicationVersionRepositoryImpl, argoClientWrapperServiceImpl, acdConfig) deploymentServiceTypeConfig, err := service.GetDeploymentServiceTypeConfig() if err != nil { return nil, err } - appStoreDeploymentServiceImpl := service.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, clusterInstalledAppsRepositoryImpl, appRepositoryImpl, appStoreDeploymentHelmServiceImpl, appStoreDeploymentArgoCdServiceImpl, environmentServiceImpl, clusterServiceImplExtended, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, globalEnvVariables, installedAppVersionHistoryRepositoryImpl, gitOpsConfigRepositoryImpl, attributesServiceImpl, deploymentServiceTypeConfig, chartTemplateServiceImpl, acdConfig) + appStoreDeploymentServiceImpl := service.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, clusterInstalledAppsRepositoryImpl, appRepositoryImpl, appStoreDeploymentHelmServiceImpl, appStoreDeploymentArgoCdServiceImpl, environmentServiceImpl, clusterServiceImplExtended, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, installedAppVersionHistoryRepositoryImpl, gitOpsConfigRepositoryImpl, attributesServiceImpl, deploymentServiceTypeConfig, chartTemplateServiceImpl, acdConfig) k8sCommonServiceImpl := k8s2.NewK8sCommonServiceImpl(sugaredLogger, k8sUtil, clusterServiceImplExtended) - manifestPushConfigRepositoryImpl := repository11.NewManifestPushConfigRepository(sugaredLogger, db) + manifestPushConfigRepositoryImpl := repository12.NewManifestPushConfigRepository(sugaredLogger, db) gitOpsManifestPushServiceImpl := app2.NewGitOpsManifestPushServiceImpl(sugaredLogger, chartTemplateServiceImpl, chartServiceImpl, gitOpsConfigRepositoryImpl, gitFactory, pipelineStatusTimelineServiceImpl, pipelineStatusTimelineRepositoryImpl, acdConfig) appServiceImpl := app2.NewAppService(envConfigOverrideRepositoryImpl, pipelineOverrideRepositoryImpl, mergeUtil, sugaredLogger, ciArtifactRepositoryImpl, pipelineRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, applicationServiceClientImpl, tokenCache, acdAuthConfig, enforcerImpl, enforcerUtilImpl, userServiceImpl, appListingRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, chartRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, imageScanDeployInfoRepositoryImpl, imageScanHistoryRepositoryImpl, argoK8sClientImpl, gitFactory, pipelineStrategyHistoryServiceImpl, configMapHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, chartTemplateServiceImpl, refChartDir, chartRefRepositoryImpl, chartServiceImpl, helmAppClientImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, appCrudOperationServiceImpl, configMapHistoryRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, dockerRegistryIpsConfigServiceImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, gitOpsConfigRepositoryImpl, appStatusServiceImpl, installedAppRepositoryImpl, appStoreDeploymentServiceImpl, k8sCommonServiceImpl, installedAppVersionHistoryRepositoryImpl, globalEnvVariables, helmAppServiceImpl, manifestPushConfigRepositoryImpl, gitOpsManifestPushServiceImpl, argoClientWrapperServiceImpl, scopedVariableCMCSManagerImpl, acdConfig) validate, err := util.IntValidator() @@ -454,8 +455,8 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - pipelineStageRepositoryImpl := repository11.NewPipelineStageRepository(sugaredLogger, db) - globalPluginRepositoryImpl := repository12.NewGlobalPluginRepository(sugaredLogger, db) + pipelineStageRepositoryImpl := repository12.NewPipelineStageRepository(sugaredLogger, db) + globalPluginRepositoryImpl := repository13.NewGlobalPluginRepository(sugaredLogger, db) pipelineStageServiceImpl := pipeline.NewPipelineStageService(sugaredLogger, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, pipelineRepositoryImpl, scopedVariableManagerImpl) globalPluginServiceImpl := plugin.NewGlobalPluginService(sugaredLogger, globalPluginRepositoryImpl, pipelineStageRepositoryImpl) dockerRegistryConfigImpl := pipeline.NewDockerRegistryConfigImpl(sugaredLogger, helmAppServiceImpl, dockerArtifactStoreRepositoryImpl, dockerRegistryIpsConfigRepositoryImpl, ociRegistryConfigRepositoryImpl) @@ -493,7 +494,7 @@ func InitializeApp() (*App, error) { buildPipelineSwitchServiceImpl := pipeline.NewBuildPipelineSwitchServiceImpl(sugaredLogger, ciPipelineRepositoryImpl, ciCdPipelineOrchestratorImpl, pipelineRepositoryImpl, ciWorkflowRepositoryImpl, appWorkflowRepositoryImpl, ciPipelineHistoryServiceImpl, ciTemplateOverrideRepositoryImpl, ciPipelineMaterialRepositoryImpl) ciPipelineConfigServiceImpl := pipeline.NewCiPipelineConfigServiceImpl(sugaredLogger, ciCdPipelineOrchestratorImpl, dockerArtifactStoreRepositoryImpl, materialRepositoryImpl, appRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ecrConfig, appWorkflowRepositoryImpl, ciCdConfig, attributesServiceImpl, pipelineStageServiceImpl, ciPipelineMaterialRepositoryImpl, ciTemplateServiceImpl, ciTemplateOverrideRepositoryImpl, ciTemplateHistoryServiceImpl, enforcerUtilImpl, ciWorkflowRepositoryImpl, resourceGroupServiceImpl, customTagServiceImpl, ciPipelineHistoryServiceImpl, cdWorkflowRepositoryImpl, buildPipelineSwitchServiceImpl) ciMaterialConfigServiceImpl := pipeline.NewCiMaterialConfigServiceImpl(sugaredLogger, materialRepositoryImpl, ciTemplateServiceImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, gitMaterialHistoryServiceImpl, pipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl) - imageTaggingRepositoryImpl := repository13.NewImageTaggingRepositoryImpl(db) + imageTaggingRepositoryImpl := repository14.NewImageTaggingRepositoryImpl(db) imageTaggingServiceImpl := pipeline.NewImageTaggingServiceImpl(imageTaggingRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, sugaredLogger) chartDeploymentServiceImpl := util.NewChartDeploymentServiceImpl(sugaredLogger, repositoryServiceClientImpl) propertiesConfigServiceImpl := pipeline.NewPropertiesConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, chartRefRepositoryImpl, utilMergeUtil, environmentRepositoryImpl, ciCdPipelineOrchestratorImpl, applicationServiceClientImpl, envLevelAppMetricsRepositoryImpl, appLevelMetricsRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl) @@ -540,7 +541,7 @@ func InitializeApp() (*App, error) { pipelineConfigRouterImpl := router.NewPipelineRouterImpl(pipelineConfigRestHandlerImpl, appWorkflowRestHandlerImpl, webhookDataRestHandlerImpl, pipelineHistoryRestHandlerImpl, pipelineStatusTimelineRestHandlerImpl) appStoreVersionValuesRepositoryImpl := appStoreValuesRepository.NewAppStoreVersionValuesRepositoryImpl(sugaredLogger, db) appStoreValuesServiceImpl := service2.NewAppStoreValuesServiceImpl(sugaredLogger, appStoreApplicationVersionRepositoryImpl, installedAppRepositoryImpl, appStoreVersionValuesRepositoryImpl, userServiceImpl) - k8sResourceHistoryRepositoryImpl := repository14.NewK8sResourceHistoryRepositoryImpl(db, sugaredLogger) + k8sResourceHistoryRepositoryImpl := repository15.NewK8sResourceHistoryRepositoryImpl(db, sugaredLogger) k8sResourceHistoryServiceImpl := kubernetesResourceAuditLogs.Newk8sResourceHistoryServiceImpl(k8sResourceHistoryRepositoryImpl, sugaredLogger, appRepositoryImpl, environmentRepositoryImpl) ephemeralContainersRepositoryImpl := repository2.NewEphemeralContainersRepositoryImpl(db) ephemeralContainerServiceImpl := cluster2.NewEphemeralContainerServiceImpl(ephemeralContainersRepositoryImpl, sugaredLogger) @@ -549,7 +550,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - installedAppServiceImpl, err := service.NewInstalledAppServiceImpl(sugaredLogger, installedAppRepositoryImpl, chartTemplateServiceImpl, refChartProxyDir, repositoryServiceClientImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, teamRepositoryImpl, appRepositoryImpl, applicationServiceClientImpl, appStoreValuesServiceImpl, pubSubClientServiceImpl, tokenCache, chartGroupDeploymentRepositoryImpl, environmentServiceImpl, argoK8sClientImpl, gitFactory, acdAuthConfig, gitOpsConfigRepositoryImpl, userServiceImpl, appStoreDeploymentFullModeServiceImpl, appStoreDeploymentServiceImpl, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, helmAppClientImpl, helmAppServiceImpl, attributesRepositoryImpl, appStatusServiceImpl, k8sUtil, pipelineStatusTimelineServiceImpl, appStoreDeploymentCommonServiceImpl, appStoreDeploymentArgoCdServiceImpl, k8sCommonServiceImpl, k8sApplicationServiceImpl, acdConfig) + installedAppServiceImpl, err := service.NewInstalledAppServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, teamRepositoryImpl, appRepositoryImpl, applicationServiceClientImpl, appStoreValuesServiceImpl, pubSubClientServiceImpl, chartGroupDeploymentRepositoryImpl, environmentServiceImpl, gitFactory, acdAuthConfig, gitOpsConfigRepositoryImpl, userServiceImpl, appStoreDeploymentFullModeServiceImpl, appStoreDeploymentServiceImpl, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, helmAppClientImpl, helmAppServiceImpl, appStatusServiceImpl, k8sUtil, pipelineStatusTimelineServiceImpl, appStoreDeploymentCommonServiceImpl, k8sCommonServiceImpl, k8sApplicationServiceImpl, acdConfig) if err != nil { return nil, err } @@ -667,9 +668,9 @@ func InitializeApp() (*App, error) { workflowActionImpl := batch.NewWorkflowActionImpl(sugaredLogger, appRepositoryImpl, appWorkflowServiceImpl, buildActionImpl, deploymentActionImpl) batchOperationRestHandlerImpl := restHandler.NewBatchOperationRestHandlerImpl(userServiceImpl, enforcerImpl, workflowActionImpl, teamServiceImpl, sugaredLogger, enforcerUtilImpl, argoUserServiceImpl) batchOperationRouterImpl := router.NewBatchOperationRouterImpl(batchOperationRestHandlerImpl, sugaredLogger) - chartGroupEntriesRepositoryImpl := repository3.NewChartGroupEntriesRepositoryImpl(db, sugaredLogger) - chartGroupReposotoryImpl := repository3.NewChartGroupReposotoryImpl(db, sugaredLogger) - chartGroupServiceImpl := service.NewChartGroupServiceImpl(chartGroupEntriesRepositoryImpl, chartGroupReposotoryImpl, sugaredLogger, chartGroupDeploymentRepositoryImpl, installedAppRepositoryImpl, appStoreVersionValuesRepositoryImpl, userAuthServiceImpl) + chartGroupEntriesRepositoryImpl := repository11.NewChartGroupEntriesRepositoryImpl(db, sugaredLogger) + chartGroupReposotoryImpl := repository11.NewChartGroupReposotoryImpl(db, sugaredLogger) + chartGroupServiceImpl := chartGroup.NewChartGroupServiceImpl(chartGroupEntriesRepositoryImpl, chartGroupReposotoryImpl, sugaredLogger, chartGroupDeploymentRepositoryImpl, installedAppRepositoryImpl, appStoreVersionValuesRepositoryImpl, userAuthServiceImpl) chartGroupRestHandlerImpl := restHandler.NewChartGroupRestHandlerImpl(chartGroupServiceImpl, sugaredLogger, userServiceImpl, enforcerImpl, validate) chartGroupRouterImpl := router.NewChartGroupRouterImpl(chartGroupRestHandlerImpl) scanToolExecutionHistoryMappingRepositoryImpl := security.NewScanToolExecutionHistoryMappingRepositoryImpl(db, sugaredLogger) From 9e36312c0a677ee7376f3ed69eea76e38aa01a68 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Mon, 15 Jan 2024 18:00:51 +0530 Subject: [PATCH 06/64] chart ref refactoring --- Wire.go | 12 +- api/chartRepo/ChartRepositoryRestHandler.go | 7 +- api/deployment/DeploymentConfigRestHandler.go | 47 +- api/restHandler/ChartRefRestHandler.go | 15 +- .../app/DeploymentPipelineRestHandler.go | 9 +- .../app/PipelineConfigRestHandler.go | 10 +- cmd/external-app/wire.go | 6 - cmd/external-app/wire_gen.go | 17 +- internal/util/ChartService.go | 28 +- pkg/app/AppService.go | 139 +---- pkg/app/ManifestPushService.go | 12 +- pkg/app/integrationTest/AppService_test.go | 3 +- pkg/appStore/bean/bean.go | 4 + .../common/AppStoreDeploymentCommonService.go | 5 +- .../AppStoreDeploymentFullModeService.go | 6 +- .../deployment/service/InstalledAppService.go | 16 +- .../service/InstalledAppService_test.go | 9 +- pkg/bulkAction/BulkUpdateService.go | 57 +- pkg/chart/ChartCompatibility_test.go | 18 +- pkg/chart/ChartService.go | 558 ++---------------- pkg/chart/ChartUtils.go | 21 +- pkg/chart/bean.go | 5 - .../repository/ChartRefRepository.go | 5 +- .../deployedAppMetrics/DeployedAppMetrics.go | 12 +- .../chartRef}/ChartCompatibility.go | 14 +- .../chartRef/ChartRefService.go | 494 ++++++++++++++++ .../chartRef/adapter/adapter.go | 45 ++ .../deploymentTemplate/chartRef/bean/bean.go | 95 +++ .../manifest/wire_deployment_manifest.go | 17 + .../DeployementTemplateService.go | 7 +- pkg/pipeline/DeploymentConfigService.go | 17 +- pkg/pipeline/PropertiesConfig.go | 14 - pkg/pipeline/WorkflowDagExecutor.go | 43 +- .../DeploymentTemplateHistoryService.go | 46 +- wire_gen.go | 256 ++++---- 35 files changed, 989 insertions(+), 1080 deletions(-) rename pkg/{chart => deployment/manifest/deploymentTemplate/chartRef}/ChartCompatibility.go (67%) create mode 100644 pkg/deployment/manifest/deploymentTemplate/chartRef/ChartRefService.go create mode 100644 pkg/deployment/manifest/deploymentTemplate/chartRef/adapter/adapter.go create mode 100644 pkg/deployment/manifest/deploymentTemplate/chartRef/bean/bean.go create mode 100644 pkg/deployment/manifest/wire_deployment_manifest.go diff --git a/Wire.go b/Wire.go index b4ef34b994..0625125515 100644 --- a/Wire.go +++ b/Wire.go @@ -85,7 +85,6 @@ import ( "github.com/devtron-labs/devtron/pkg/appClone" "github.com/devtron-labs/devtron/pkg/appClone/batch" "github.com/devtron-labs/devtron/pkg/appStatus" - appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" appStoreDeploymentFullMode "github.com/devtron-labs/devtron/pkg/appStore/deployment/fullMode" repository4 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" "github.com/devtron-labs/devtron/pkg/appStore/deployment/service" @@ -97,7 +96,7 @@ import ( chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" "github.com/devtron-labs/devtron/pkg/commonService" delete2 "github.com/devtron-labs/devtron/pkg/delete" - "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" + "github.com/devtron-labs/devtron/pkg/deployment/manifest" "github.com/devtron-labs/devtron/pkg/deploymentGroup" "github.com/devtron-labs/devtron/pkg/devtronResource" repository9 "github.com/devtron-labs/devtron/pkg/devtronResource/repository" @@ -158,6 +157,8 @@ func InitializeApp() (*App, error) { apiToken.ApiTokenWireSet, webhookHelm.WebhookHelmWireSet, terminal.TerminalWireSet, + manifest.DeploymentManifestWireSet, + // -------wireset end ---------- //------- gitSensor.GetConfig, @@ -171,10 +172,6 @@ func InitializeApp() (*App, error) { //sql.NewDbConnection, //app.GetACDAuthConfig, util3.GetACDAuthConfig, - wire.Value(chartRepoRepository.RefChartDir("scripts/devtron-reference-helm-charts")), - wire.Value(appStoreBean.RefChartProxyDir("scripts/devtron-reference-helm-charts")), - wire.Value(chart.DefaultChart("reference-app-rolling")), - wire.Value(util.ChartWorkingDir("/tmp/charts/")), connection.SettingsManager, //auth.GetConfig, @@ -290,9 +287,6 @@ func InitializeApp() (*App, error) { //end - deployedAppMetrics.NewDeployedAppMetricsServiceImpl, - wire.Bind(new(deployedAppMetrics.DeployedAppMetricsService), new(*deployedAppMetrics.DeployedAppMetricsServiceImpl)), - chart.NewChartServiceImpl, wire.Bind(new(chart.ChartService), new(*chart.ChartServiceImpl)), bulkAction.NewBulkUpdateServiceImpl, diff --git a/api/chartRepo/ChartRepositoryRestHandler.go b/api/chartRepo/ChartRepositoryRestHandler.go index 495fa5b2bb..b7c0486d5f 100644 --- a/api/chartRepo/ChartRepositoryRestHandler.go +++ b/api/chartRepo/ChartRepositoryRestHandler.go @@ -30,7 +30,6 @@ import ( "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" "github.com/devtron-labs/devtron/pkg/auth/user" "github.com/devtron-labs/devtron/pkg/chartRepo" - chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" delete2 "github.com/devtron-labs/devtron/pkg/delete" "github.com/gorilla/mux" "go.uber.org/zap" @@ -64,14 +63,12 @@ type ChartRepositoryRestHandlerImpl struct { enforcer casbin.Enforcer validator *validator.Validate deleteService delete2.DeleteService - chartRefRepository chartRepoRepository.ChartRefRepository - refChartDir chartRepoRepository.RefChartDir attributesService attributes.AttributesService } func NewChartRepositoryRestHandlerImpl(Logger *zap.SugaredLogger, userAuthService user.UserService, chartRepositoryService chartRepo.ChartRepositoryService, enforcer casbin.Enforcer, validator *validator.Validate, deleteService delete2.DeleteService, - chartRefRepository chartRepoRepository.ChartRefRepository, refChartDir chartRepoRepository.RefChartDir, attributesService attributes.AttributesService) *ChartRepositoryRestHandlerImpl { + attributesService attributes.AttributesService) *ChartRepositoryRestHandlerImpl { return &ChartRepositoryRestHandlerImpl{ Logger: Logger, chartRepositoryService: chartRepositoryService, @@ -79,8 +76,6 @@ func NewChartRepositoryRestHandlerImpl(Logger *zap.SugaredLogger, userAuthServic enforcer: enforcer, validator: validator, deleteService: deleteService, - chartRefRepository: chartRefRepository, - refChartDir: refChartDir, attributesService: attributesService, } } diff --git a/api/deployment/DeploymentConfigRestHandler.go b/api/deployment/DeploymentConfigRestHandler.go index 53be9a4b6a..d9d819959e 100644 --- a/api/deployment/DeploymentConfigRestHandler.go +++ b/api/deployment/DeploymentConfigRestHandler.go @@ -3,6 +3,8 @@ package deployment import ( "encoding/json" "fmt" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/bean" "io/ioutil" "net/http" "os" @@ -20,7 +22,6 @@ import ( "github.com/gorilla/mux" "github.com/juju/errors" "go.uber.org/zap" - "gopkg.in/go-playground/validator.v9" ) type DeploymentConfigRestHandler interface { @@ -34,10 +35,8 @@ type DeploymentConfigRestHandlerImpl struct { Logger *zap.SugaredLogger userAuthService user.UserService enforcer casbin.Enforcer - validator *validator.Validate - refChartDir chartRepoRepository.RefChartDir - chartService chart.ChartService - chartRefRepository chartRepoRepository.ChartRefRepository + chartService chart.ChartService + chartRefService chartRef.ChartRefService } type DeploymentChartInfo struct { @@ -49,16 +48,14 @@ type DeploymentChartInfo struct { Message string `json:"message"` } -func NewDeploymentConfigRestHandlerImpl(Logger *zap.SugaredLogger, userAuthService user.UserService, enforcer casbin.Enforcer, validator *validator.Validate, - refChartDir chartRepoRepository.RefChartDir, chartService chart.ChartService, chartRefRepository chartRepoRepository.ChartRefRepository) *DeploymentConfigRestHandlerImpl { +func NewDeploymentConfigRestHandlerImpl(Logger *zap.SugaredLogger, userAuthService user.UserService, enforcer casbin.Enforcer, + chartService chart.ChartService, chartRefService chartRef.ChartRefService) *DeploymentConfigRestHandlerImpl { return &DeploymentConfigRestHandlerImpl{ - Logger: Logger, - userAuthService: userAuthService, - enforcer: enforcer, - validator: validator, - refChartDir: refChartDir, - chartService: chartService, - chartRefRepository: chartRefRepository, + Logger: Logger, + userAuthService: userAuthService, + enforcer: enforcer, + chartService: chartService, + chartRefService: chartRefService, } } @@ -102,7 +99,7 @@ func (handler *DeploymentConfigRestHandlerImpl) CreateChartFromFile(w http.Respo return } - chartInfo, err := handler.chartService.ExtractChartIfMissing(fileBytes, string(handler.refChartDir), "") + chartInfo, err := handler.chartRefService.ExtractChartIfMissing(fileBytes, chartRepoRepository.RefChartDirPath, "") if err != nil { if chartInfo != nil && chartInfo.TemporaryFolder != "" { @@ -111,7 +108,7 @@ func (handler *DeploymentConfigRestHandlerImpl) CreateChartFromFile(w http.Respo handler.Logger.Errorw("error in deleting temp dir ", "err", err1) } } - if err.Error() == chart.CHART_ALREADY_EXISTS_INTERNAL_ERROR || err.Error() == chart.CHART_NAME_RESERVED_INTERNAL_ERROR { + if err.Error() == bean.CHART_ALREADY_EXISTS_INTERNAL_ERROR || err.Error() == bean.CHART_NAME_RESERVED_INTERNAL_ERROR { common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } @@ -119,7 +116,7 @@ func (handler *DeploymentConfigRestHandlerImpl) CreateChartFromFile(w http.Respo return } - chartRefs := &chartRepoRepository.ChartRef{ + chartRefs := &bean.CustomChartRefDto{ Name: chartInfo.ChartName, Version: chartInfo.ChartVersion, Location: chartInfo.ChartLocation, @@ -178,7 +175,7 @@ func (handler *DeploymentConfigRestHandlerImpl) SaveChart(w http.ResponseWriter, return } - location := filepath.Join(string(handler.refChartDir), request.FileId) + location := filepath.Join(chartRepoRepository.RefChartDirPath, request.FileId) if request.Action == "Save" { file, err := ioutil.ReadFile(filepath.Join(location, "output.json")) if err != nil { @@ -186,17 +183,17 @@ func (handler *DeploymentConfigRestHandlerImpl) SaveChart(w http.ResponseWriter, common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - chartRefs := &chartRepoRepository.ChartRef{} - err = json.Unmarshal(file, &chartRefs) + customChartRefDto := &bean.CustomChartRefDto{} + err = json.Unmarshal(file, &customChartRefDto) if err != nil { handler.Logger.Errorw("unmarshall err", "err", err) common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - chartRefs.ChartDescription = request.Description - err = handler.chartRefRepository.Save(chartRefs) + customChartRefDto.ChartDescription = request.Description + err = handler.chartRefService.SaveCustomChart(customChartRefDto) if err != nil { - handler.Logger.Errorw("error in saving Chart", "err", err) + handler.Logger.Errorw("error in saving Chart", "err", err, "request", customChartRefDto) common.WriteJsonResp(w, err, "Chart couldn't be saved", http.StatusInternalServerError) return } @@ -234,7 +231,7 @@ func (handler *DeploymentConfigRestHandlerImpl) DownloadChart(w http.ResponseWri common.WriteJsonResp(w, fmt.Errorf("error in parsing chartRefId : %s must be integer", chartRefId), nil, http.StatusBadRequest) return } - manifestByteArr, err := handler.chartService.GetCustomChartInBytes(chartRefId) + manifestByteArr, err := handler.chartRefService.GetCustomChartInBytes(chartRefId) if err != nil { handler.Logger.Errorw("error in converting chart to bytes", "err", err) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) @@ -258,7 +255,7 @@ func (handler *DeploymentConfigRestHandlerImpl) GetUploadedCharts(w http.Respons return } - charts, err := handler.chartService.FetchCustomChartsInfo() + charts, err := handler.chartRefService.FetchCustomChartsInfo() if err != nil { common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return diff --git a/api/restHandler/ChartRefRestHandler.go b/api/restHandler/ChartRefRestHandler.go index 9e85cdc09c..decebfbfe7 100644 --- a/api/restHandler/ChartRefRestHandler.go +++ b/api/restHandler/ChartRefRestHandler.go @@ -19,7 +19,8 @@ package restHandler import ( "github.com/devtron-labs/devtron/api/restHandler/common" - "github.com/devtron-labs/devtron/pkg/chart" + chartService "github.com/devtron-labs/devtron/pkg/chart" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" "github.com/gorilla/mux" "go.uber.org/zap" "net/http" @@ -33,17 +34,19 @@ type ChartRefRestHandler interface { } type ChartRefRestHandlerImpl struct { - chartService chart.ChartService - logger *zap.SugaredLogger + logger *zap.SugaredLogger + chartRefService chartRef.ChartRefService + chartService chartService.ChartService } -func NewChartRefRestHandlerImpl(chartService chart.ChartService, logger *zap.SugaredLogger) *ChartRefRestHandlerImpl { - handler := &ChartRefRestHandlerImpl{chartService: chartService, logger: logger} +func NewChartRefRestHandlerImpl(logger *zap.SugaredLogger, chartRefService chartRef.ChartRefService, + chartService chartService.ChartService) *ChartRefRestHandlerImpl { + handler := &ChartRefRestHandlerImpl{logger: logger, chartRefService: chartRefService, chartService: chartService} return handler } func (handler ChartRefRestHandlerImpl) ChartRefAutocomplete(w http.ResponseWriter, r *http.Request) { - result, err := handler.chartService.ChartRefAutocomplete() + result, err := handler.chartRefService.ChartRefAutocomplete() if err != nil { handler.logger.Errorw("service err, ChartRefAutocomplete", "err", err) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) diff --git a/api/restHandler/app/DeploymentPipelineRestHandler.go b/api/restHandler/app/DeploymentPipelineRestHandler.go index 6516880e78..8889f2d5ea 100644 --- a/api/restHandler/app/DeploymentPipelineRestHandler.go +++ b/api/restHandler/app/DeploymentPipelineRestHandler.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + bean4 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/bean" "io" "net/http" "strconv" @@ -185,7 +186,7 @@ func (handler PipelineConfigRestHandlerImpl) CreateCdPipeline(w http.ResponseWri return } handler.Logger.Infow("request payload, CreateCdPipeline", "payload", cdPipeline) - userUploaded, err := handler.chartService.CheckCustomChartByAppId(cdPipeline.AppId) + userUploaded, err := handler.chartService.CheckIfChartRefUserUploadedByAppId(cdPipeline.AppId) if !userUploaded { err = handler.validator.Struct(cdPipeline) if err != nil { @@ -541,7 +542,7 @@ func (handler PipelineConfigRestHandlerImpl) ChangeChartRef(w http.ResponseWrite common.WriteJsonResp(w, err, "specific environment is not overriden", http.StatusUnprocessableEntity) return } - compatible, oldChartType, newChartType := handler.chartService.ChartRefIdsCompatible(envConfigProperties.ChartRefId, request.TargetChartRefId) + compatible, oldChartType, newChartType := handler.chartRefService.ChartRefIdsCompatible(envConfigProperties.ChartRefId, request.TargetChartRefId) if !compatible { common.WriteJsonResp(w, fmt.Errorf("charts not compatible"), "chart not compatible", http.StatusUnprocessableEntity) return @@ -553,7 +554,7 @@ func (handler PipelineConfigRestHandlerImpl) ChangeChartRef(w http.ResponseWrite return } - if newChartType == chart.RolloutChartType { + if newChartType == bean4.RolloutChartType { enabled, err := handler.chartService.FlaggerCanaryEnabled(envConfigProperties.EnvOverrideValues) if err != nil || enabled { handler.Logger.Errorw("rollout charts do not support flaggerCanary, ChangeChartRef", "err", err, "payload", request) @@ -989,7 +990,7 @@ func (handler PipelineConfigRestHandlerImpl) GetDeploymentTemplate(w http.Respon appConfigResponse := make(map[string]interface{}) appConfigResponse["globalConfig"] = nil - err = handler.chartService.CheckChartExists(chartRefId) + err = handler.chartRefService.CheckChartExists(chartRefId) if err != nil { handler.Logger.Errorw("refChartDir Not Found err, JsonSchemaExtractFromFile", err) common.WriteJsonResp(w, err, nil, http.StatusForbidden) diff --git a/api/restHandler/app/PipelineConfigRestHandler.go b/api/restHandler/app/PipelineConfigRestHandler.go index 13ff4f3e37..d58c9464d5 100644 --- a/api/restHandler/app/PipelineConfigRestHandler.go +++ b/api/restHandler/app/PipelineConfigRestHandler.go @@ -23,6 +23,7 @@ import ( "encoding/json" "fmt" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" "io" "net/http" "strconv" @@ -43,7 +44,6 @@ import ( "go.opentelemetry.io/otel" bean2 "github.com/devtron-labs/devtron/api/bean" - "github.com/devtron-labs/devtron/client/argocdServer/application" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/sql/repository/security" @@ -110,7 +110,6 @@ type PipelineConfigRestHandlerImpl struct { chartService chart.ChartService propertiesConfigService pipeline.PropertiesConfigService dbMigrationService pipeline.DbMigrationService - application application.ServiceClient userAuthService user.UserService validator *validator.Validate teamService team.TeamService @@ -134,13 +133,13 @@ type PipelineConfigRestHandlerImpl struct { pipelineRestHandlerEnvConfig *PipelineRestHandlerEnvConfig ciArtifactRepository repository.CiArtifactRepository deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService + chartRefService chartRef.ChartRefService } func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger *zap.SugaredLogger, chartService chart.ChartService, propertiesConfigService pipeline.PropertiesConfigService, dbMigrationService pipeline.DbMigrationService, - application application.ServiceClient, userAuthService user.UserService, teamService team.TeamService, enforcer casbin.Enforcer, @@ -159,7 +158,8 @@ func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger argoUserService argo.ArgoUserService, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, imageTaggingService pipeline.ImageTaggingService, ciArtifactRepository repository.CiArtifactRepository, - deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService) *PipelineConfigRestHandlerImpl { + deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService, + chartRefService chartRef.ChartRefService) *PipelineConfigRestHandlerImpl { envConfig := &PipelineRestHandlerEnvConfig{} err := env.Parse(envConfig) if err != nil { @@ -171,7 +171,6 @@ func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger chartService: chartService, propertiesConfigService: propertiesConfigService, dbMigrationService: dbMigrationService, - application: application, userAuthService: userAuthService, validator: validator, teamService: teamService, @@ -198,6 +197,7 @@ func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger pipelineRestHandlerEnvConfig: envConfig, ciArtifactRepository: ciArtifactRepository, deployedAppMetricsService: deployedAppMetricsService, + chartRefService: chartRefService, } } diff --git a/cmd/external-app/wire.go b/cmd/external-app/wire.go index 044f9c0d98..45c6a0f225 100644 --- a/cmd/external-app/wire.go +++ b/cmd/external-app/wire.go @@ -38,12 +38,10 @@ import ( security2 "github.com/devtron-labs/devtron/internal/sql/repository/security" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app" - appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" repository3 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" appStoreDeploymentTool "github.com/devtron-labs/devtron/pkg/appStore/deployment/tool" appStoreDeploymentGitopsTool "github.com/devtron-labs/devtron/pkg/appStore/deployment/tool/gitops" "github.com/devtron-labs/devtron/pkg/attributes" - chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" delete2 "github.com/devtron-labs/devtron/pkg/delete" "github.com/devtron-labs/devtron/pkg/kubernetesResourceAuditLogs" repository2 "github.com/devtron-labs/devtron/pkg/kubernetesResourceAuditLogs/repository" @@ -140,8 +138,6 @@ func InitializeApp() (*App, error) { // binding gitops to helm (for hyperion) wire.Bind(new(appStoreDeploymentGitopsTool.AppStoreDeploymentArgoCdService), new(*appStoreDeploymentTool.AppStoreDeploymentHelmServiceImpl)), - wire.Value(chartRepoRepository.RefChartDir("scripts/devtron-reference-helm-charts")), - router.NewTelemetryRouterImpl, wire.Bind(new(router.TelemetryRouter), new(*router.TelemetryRouterImpl)), restHandler.NewTelemetryRestHandlerImpl, @@ -179,8 +175,6 @@ func InitializeApp() (*App, error) { util.NewChartTemplateServiceImpl, wire.Bind(new(util.ChartTemplateService), new(*util.ChartTemplateServiceImpl)), - wire.Value(util.ChartWorkingDir("/tmp/charts/")), - wire.Value(appStoreBean.RefChartProxyDir("scripts/devtron-reference-helm-charts")), util.NewGitFactory, util.NewGitCliUtil, diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index c95ee93e35..de881eb3d0 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -45,7 +45,6 @@ import ( "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/apiToken" app2 "github.com/devtron-labs/devtron/pkg/app" - "github.com/devtron-labs/devtron/pkg/appStore/bean" "github.com/devtron-labs/devtron/pkg/appStore/chartProvider" "github.com/devtron-labs/devtron/pkg/appStore/deployment/common" repository4 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" @@ -229,7 +228,6 @@ func InitializeApp() (*App, error) { return nil, err } dashboardRouterImpl := dashboard.NewDashboardRouterImpl(sugaredLogger, dashboardConfig) - chartWorkingDir := _wireChartWorkingDirValue gitOpsConfigRepositoryImpl := repository3.NewGitOpsConfigRepositoryImpl(sugaredLogger, db) gitCliUtil := util.NewGitCliUtil(sugaredLogger) gitFactory, err := util.NewGitFactory(sugaredLogger, gitOpsConfigRepositoryImpl, gitCliUtil) @@ -241,9 +239,8 @@ func InitializeApp() (*App, error) { return nil, err } chartRepositoryImpl := chartRepoRepository.NewChartRepository(db) - chartTemplateServiceImpl := util.NewChartTemplateServiceImpl(sugaredLogger, chartWorkingDir, httpClient, gitFactory, globalEnvVariables, gitOpsConfigRepositoryImpl, userRepositoryImpl, chartRepositoryImpl) - refChartProxyDir := _wireRefChartProxyDirValue - appStoreDeploymentCommonServiceImpl := appStoreDeploymentCommon.NewAppStoreDeploymentCommonServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, chartTemplateServiceImpl, refChartProxyDir, gitFactory, gitOpsConfigRepositoryImpl) + chartTemplateServiceImpl := util.NewChartTemplateServiceImpl(sugaredLogger, gitFactory, globalEnvVariables, gitOpsConfigRepositoryImpl, userRepositoryImpl, chartRepositoryImpl) + appStoreDeploymentCommonServiceImpl := appStoreDeploymentCommon.NewAppStoreDeploymentCommonServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, chartTemplateServiceImpl, gitFactory, gitOpsConfigRepositoryImpl) attributesServiceImpl := attributes.NewAttributesServiceImpl(sugaredLogger, attributesRepositoryImpl) helmAppRestHandlerImpl := client2.NewHelmAppRestHandlerImpl(sugaredLogger, helmAppServiceImpl, enforcerImpl, clusterServiceImpl, enforcerUtilHelmImpl, appStoreDeploymentCommonServiceImpl, userServiceImpl, attributesServiceImpl, serverEnvConfigServerEnvConfig) helmAppRouterImpl := client2.NewHelmAppRouterImpl(helmAppRestHandlerImpl) @@ -263,9 +260,7 @@ func InitializeApp() (*App, error) { enforcerUtilImpl := rbac.NewEnforcerUtilImpl(sugaredLogger, teamRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, clusterRepositoryImpl, enforcerImpl) k8sApplicationRestHandlerImpl := application2.NewK8sApplicationRestHandlerImpl(sugaredLogger, k8sApplicationServiceImpl, pumpImpl, terminalSessionHandlerImpl, enforcerImpl, enforcerUtilHelmImpl, enforcerUtilImpl, helmAppServiceImpl, userServiceImpl, k8sCommonServiceImpl, validate) k8sApplicationRouterImpl := application2.NewK8sApplicationRouterImpl(k8sApplicationRestHandlerImpl) - chartRefRepositoryImpl := chartRepoRepository.NewChartRefRepositoryImpl(db) - refChartDir := _wireRefChartDirValue - chartRepositoryRestHandlerImpl := chartRepo2.NewChartRepositoryRestHandlerImpl(sugaredLogger, userServiceImpl, chartRepositoryServiceImpl, enforcerImpl, validate, deleteServiceImpl, chartRefRepositoryImpl, refChartDir, attributesServiceImpl) + chartRepositoryRestHandlerImpl := chartRepo2.NewChartRepositoryRestHandlerImpl(sugaredLogger, userServiceImpl, chartRepositoryServiceImpl, enforcerImpl, validate, deleteServiceImpl, attributesServiceImpl) chartRepositoryRouterImpl := chartRepo2.NewChartRepositoryRouterImpl(chartRepositoryRestHandlerImpl) appStoreServiceImpl := service.NewAppStoreServiceImpl(sugaredLogger, appStoreApplicationVersionRepositoryImpl) appStoreRestHandlerImpl := appStoreDiscover.NewAppStoreRestHandlerImpl(sugaredLogger, userServiceImpl, appStoreServiceImpl, enforcerImpl) @@ -384,9 +379,3 @@ func InitializeApp() (*App, error) { mainApp := NewApp(db, sessionManager, muxRouter, telemetryEventClientImpl, posthogClient, sugaredLogger) return mainApp, nil } - -var ( - _wireChartWorkingDirValue = util.ChartWorkingDir("/tmp/charts/") - _wireRefChartProxyDirValue = appStoreBean.RefChartProxyDir("scripts/devtron-reference-helm-charts") - _wireRefChartDirValue = chartRepoRepository.RefChartDir("scripts/devtron-reference-helm-charts") -) diff --git a/internal/util/ChartService.go b/internal/util/ChartService.go index e3329be2f5..2deb276c30 100644 --- a/internal/util/ChartService.go +++ b/internal/util/ChartService.go @@ -23,7 +23,6 @@ import ( "fmt" "io/ioutil" "math/rand" - "net/http" "os" "path/filepath" "regexp" @@ -48,11 +47,12 @@ import ( "sigs.k8s.io/yaml" ) -type ChartWorkingDir string - -const PIPELINE_DEPLOYMENT_TYPE_ACD string = "argo_cd" -const PIPELINE_DEPLOYMENT_TYPE_HELM string = "helm" -const PIPELINE_DEPLOYMENT_TYPE_MANIFEST_DOWNLOAD string = "manifest_download" +const ( + PIPELINE_DEPLOYMENT_TYPE_ACD = "argo_cd" + PIPELINE_DEPLOYMENT_TYPE_HELM = "helm" + PIPELINE_DEPLOYMENT_TYPE_MANIFEST_DOWNLOAD = "manifest_download" + ChartWorkingDirPath = "/tmp/charts/" +) type ChartCreateRequest struct { ChartMetaData *chart.Metadata @@ -89,9 +89,7 @@ type ChartTemplateService interface { type ChartTemplateServiceImpl struct { randSource rand.Source logger *zap.SugaredLogger - chartWorkingDir ChartWorkingDir gitFactory *GitFactory - client *http.Client globalEnvVariables *util.GlobalEnvVariables gitOpsConfigRepository repository.GitOpsConfigRepository userRepository repository2.UserRepository @@ -108,16 +106,12 @@ type ChartValues struct { } func NewChartTemplateServiceImpl(logger *zap.SugaredLogger, - chartWorkingDir ChartWorkingDir, - client *http.Client, gitFactory *GitFactory, globalEnvVariables *util.GlobalEnvVariables, gitOpsConfigRepository repository.GitOpsConfigRepository, userRepository repository2.UserRepository, chartRepository chartRepoRepository.ChartRepository) *ChartTemplateServiceImpl { return &ChartTemplateServiceImpl{ randSource: rand.NewSource(time.Now().UnixNano()), logger: logger, - chartWorkingDir: chartWorkingDir, - client: client, gitFactory: gitFactory, globalEnvVariables: globalEnvVariables, gitOpsConfigRepository: gitOpsConfigRepository, @@ -154,7 +148,7 @@ func (impl ChartTemplateServiceImpl) GetChartVersion(location string) (string, e func (impl ChartTemplateServiceImpl) FetchValuesFromReferenceChart(chartMetaData *chart.Metadata, refChartLocation string, templateName string, userId int32, pipelineStrategyPath string) (*ChartValues, *ChartGitAttribute, error) { chartMetaData.ApiVersion = "v1" // ensure always v1 dir := impl.GetDir() - chartDir := filepath.Join(string(impl.chartWorkingDir), dir) + chartDir := filepath.Join(ChartWorkingDirPath, dir) impl.logger.Debugw("chart dir ", "chart", chartMetaData.Name, "dir", chartDir) err := os.MkdirAll(chartDir, os.ModePerm) //hack for concurrency handling if err != nil { @@ -194,7 +188,7 @@ func (impl ChartTemplateServiceImpl) FetchValuesFromReferenceChart(chartMetaData func (impl ChartTemplateServiceImpl) BuildChart(ctx context.Context, chartMetaData *chart.Metadata, referenceTemplatePath string) (string, error) { chartMetaData.ApiVersion = "v1" // ensure always v1 dir := impl.GetDir() - tempReferenceTemplateDir := filepath.Join(string(impl.chartWorkingDir), dir) + tempReferenceTemplateDir := filepath.Join(ChartWorkingDirPath, dir) impl.logger.Debugw("chart dir ", "chart", chartMetaData.Name, "dir", tempReferenceTemplateDir) err := os.MkdirAll(tempReferenceTemplateDir, os.ModePerm) //hack for concurrency handling if err != nil { @@ -222,7 +216,7 @@ func (impl ChartTemplateServiceImpl) BuildChartProxyForHelmApps(chartCreateReque chartMetaData := chartCreateRequest.ChartMetaData chartMetaData.ApiVersion = "v2" // ensure always v2 dir := impl.GetDir() - chartDir := filepath.Join(string(impl.chartWorkingDir), dir) + chartDir := filepath.Join(ChartWorkingDirPath, dir) impl.logger.Debugw("chart dir ", "chart", chartMetaData.Name, "dir", chartDir) err := os.MkdirAll(chartDir, os.ModePerm) //hack for concurrency handling if err != nil { @@ -488,7 +482,7 @@ func (impl ChartTemplateServiceImpl) GetDir() string { func (impl ChartTemplateServiceImpl) CreateChartProxy(chartMetaData *chart.Metadata, refChartLocation string, envName string, installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (string, *ChartGitAttribute, error) { chartMetaData.ApiVersion = "v2" // ensure always v2 dir := impl.GetDir() - chartDir := filepath.Join(string(impl.chartWorkingDir), dir) + chartDir := filepath.Join(ChartWorkingDirPath, dir) impl.logger.Debugw("chart dir ", "chart", chartMetaData.Name, "dir", chartDir) err := os.MkdirAll(chartDir, os.ModePerm) //hack for concurrency handling if err != nil { @@ -670,7 +664,7 @@ func (impl ChartTemplateServiceImpl) GetGitOpsRepoNameFromUrl(gitRepoUrl string) func (impl ChartTemplateServiceImpl) GetByteArrayRefChart(chartMetaData *chart.Metadata, referenceTemplatePath string) ([]byte, error) { chartMetaData.ApiVersion = "v1" // ensure always v1 dir := impl.GetDir() - tempReferenceTemplateDir := filepath.Join(string(impl.chartWorkingDir), dir) + tempReferenceTemplateDir := filepath.Join(ChartWorkingDirPath, dir) impl.logger.Debugw("chart dir ", "chart", chartMetaData.Name, "dir", tempReferenceTemplateDir) err := os.MkdirAll(tempReferenceTemplateDir, os.ModePerm) //hack for concurrency handling if err != nil { diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index 5148a4279d..5a17c7dfcd 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -21,6 +21,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" "io/ioutil" "net/url" "os" @@ -33,18 +34,9 @@ import ( "github.com/caarlos0/env" k8sCommonBean "github.com/devtron-labs/common-lib/utils/k8s/commonBean" "github.com/devtron-labs/common-lib/utils/k8s/health" - client2 "github.com/devtron-labs/devtron/api/helm-app" status2 "github.com/devtron-labs/devtron/pkg/app/status" repository4 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" - "github.com/devtron-labs/devtron/pkg/appStore/deployment/service" - "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" - "github.com/devtron-labs/devtron/pkg/auth/user" bean2 "github.com/devtron-labs/devtron/pkg/bean" - "github.com/devtron-labs/devtron/pkg/chart" - "github.com/devtron-labs/devtron/pkg/dockerRegistry" - "github.com/devtron-labs/devtron/pkg/k8s" - repository3 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" - repository5 "github.com/devtron-labs/devtron/pkg/pipeline/repository" "github.com/devtron-labs/devtron/pkg/resourceQualifiers" "github.com/devtron-labs/devtron/pkg/variables" _ "github.com/devtron-labs/devtron/pkg/variables/repository" @@ -53,14 +45,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" chart2 "k8s.io/helm/pkg/proto/hapi/chart" - "github.com/devtron-labs/devtron/internal/sql/repository/app" - "github.com/devtron-labs/devtron/pkg/appStatus" - chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" - repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" - history2 "github.com/devtron-labs/devtron/pkg/pipeline/history" - "github.com/devtron-labs/devtron/pkg/sql" - util3 "github.com/devtron-labs/devtron/pkg/util" - application2 "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" "github.com/devtron-labs/devtron/api/bean" @@ -68,14 +52,16 @@ import ( "github.com/devtron-labs/devtron/client/argocdServer/application" client "github.com/devtron-labs/devtron/client/events" "github.com/devtron-labs/devtron/internal/sql/repository" + "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/devtron-labs/devtron/internal/sql/repository/security" . "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/appStatus" + chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" "github.com/devtron-labs/devtron/pkg/commonService" + "github.com/devtron-labs/devtron/pkg/sql" util2 "github.com/devtron-labs/devtron/util" util "github.com/devtron-labs/devtron/util/event" - "github.com/devtron-labs/devtron/util/rbac" "github.com/go-pg/pg" "go.uber.org/zap" ) @@ -108,62 +94,29 @@ type AppServiceImpl struct { pipelineOverrideRepository chartConfig.PipelineOverrideRepository mergeUtil *MergeUtil logger *zap.SugaredLogger - ciArtifactRepository repository.CiArtifactRepository pipelineRepository pipelineConfig.PipelineRepository - gitFactory *GitFactory - dbMigrationConfigRepository pipelineConfig.DbMigrationConfigRepository eventClient client.EventClient eventFactory client.EventFactory acdClient application.ServiceClient - tokenCache *util3.TokenCache - acdAuthConfig *util3.ACDAuthConfig - enforcer casbin.Enforcer - enforcerUtil rbac.EnforcerUtil - user user.UserService - appListingRepository repository.AppListingRepository appRepository app.AppRepository - envRepository repository2.EnvironmentRepository - pipelineConfigRepository chartConfig.PipelineConfigRepository configMapRepository chartConfig.ConfigMapRepository chartRepository chartRepoRepository.ChartRepository - ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository cdWorkflowRepository pipelineConfig.CdWorkflowRepository commonService commonService.CommonService - imageScanDeployInfoRepository security.ImageScanDeployInfoRepository - imageScanHistoryRepository security.ImageScanHistoryRepository - ArgoK8sClient argocdServer.ArgoK8sClient - pipelineStrategyHistoryService history2.PipelineStrategyHistoryService - configMapHistoryService history2.ConfigMapHistoryService - deploymentTemplateHistoryService history2.DeploymentTemplateHistoryService chartTemplateService ChartTemplateService - refChartDir chartRepoRepository.RefChartDir - helmAppClient client2.HelmAppClient - helmAppService client2.HelmAppService - chartRefRepository chartRepoRepository.ChartRefRepository - chartService chart.ChartService argoUserService argo.ArgoUserService pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository - appCrudOperationService AppCrudOperationService - configMapHistoryRepository repository3.ConfigMapHistoryRepository - strategyHistoryRepository repository3.PipelineStrategyHistoryRepository - deploymentTemplateHistoryRepository repository3.DeploymentTemplateHistoryRepository - dockerRegistryIpsConfigService dockerRegistry.DockerRegistryIpsConfigService pipelineStatusTimelineResourcesService status2.PipelineStatusTimelineResourcesService pipelineStatusSyncDetailService status2.PipelineStatusSyncDetailService pipelineStatusTimelineService status2.PipelineStatusTimelineService appStatusConfig *AppServiceConfig - gitOpsConfigRepository repository.GitOpsConfigRepository appStatusService appStatus.AppStatusService installedAppRepository repository4.InstalledAppRepository - AppStoreDeploymentService service.AppStoreDeploymentService - K8sCommonService k8s.K8sCommonService installedAppVersionHistoryRepository repository4.InstalledAppVersionHistoryRepository globalEnvVariables *util2.GlobalEnvVariables - manifestPushConfigRepository repository5.ManifestPushConfigRepository - GitOpsManifestPushService GitOpsPushService scopedVariableManager variables.ScopedVariableCMCSManager - argoClientWrapperService argocdServer.ArgoClientWrapperService acdConfig *argocdServer.ACDConfig + chartRefService chartRef.ChartRefService } type AppService interface { @@ -196,119 +149,55 @@ func NewAppService( pipelineOverrideRepository chartConfig.PipelineOverrideRepository, mergeUtil *MergeUtil, logger *zap.SugaredLogger, - ciArtifactRepository repository.CiArtifactRepository, pipelineRepository pipelineConfig.PipelineRepository, - dbMigrationConfigRepository pipelineConfig.DbMigrationConfigRepository, eventClient client.EventClient, eventFactory client.EventFactory, acdClient application.ServiceClient, - cache *util3.TokenCache, authConfig *util3.ACDAuthConfig, - enforcer casbin.Enforcer, enforcerUtil rbac.EnforcerUtil, user user.UserService, - appListingRepository repository.AppListingRepository, appRepository app.AppRepository, - envRepository repository2.EnvironmentRepository, - pipelineConfigRepository chartConfig.PipelineConfigRepository, configMapRepository chartConfig.ConfigMapRepository, chartRepository chartRepoRepository.ChartRepository, - ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, commonService commonService.CommonService, - imageScanDeployInfoRepository security.ImageScanDeployInfoRepository, - imageScanHistoryRepository security.ImageScanHistoryRepository, - ArgoK8sClient argocdServer.ArgoK8sClient, - gitFactory *GitFactory, - pipelineStrategyHistoryService history2.PipelineStrategyHistoryService, - configMapHistoryService history2.ConfigMapHistoryService, - deploymentTemplateHistoryService history2.DeploymentTemplateHistoryService, chartTemplateService ChartTemplateService, - refChartDir chartRepoRepository.RefChartDir, - chartRefRepository chartRepoRepository.ChartRefRepository, - chartService chart.ChartService, - helmAppClient client2.HelmAppClient, argoUserService argo.ArgoUserService, cdPipelineStatusTimelineRepo pipelineConfig.PipelineStatusTimelineRepository, - appCrudOperationService AppCrudOperationService, - configMapHistoryRepository repository3.ConfigMapHistoryRepository, - strategyHistoryRepository repository3.PipelineStrategyHistoryRepository, - deploymentTemplateHistoryRepository repository3.DeploymentTemplateHistoryRepository, - dockerRegistryIpsConfigService dockerRegistry.DockerRegistryIpsConfigService, pipelineStatusTimelineResourcesService status2.PipelineStatusTimelineResourcesService, pipelineStatusSyncDetailService status2.PipelineStatusSyncDetailService, pipelineStatusTimelineService status2.PipelineStatusTimelineService, appStatusConfig *AppServiceConfig, - gitOpsConfigRepository repository.GitOpsConfigRepository, appStatusService appStatus.AppStatusService, installedAppRepository repository4.InstalledAppRepository, - AppStoreDeploymentService service.AppStoreDeploymentService, - k8sCommonService k8s.K8sCommonService, installedAppVersionHistoryRepository repository4.InstalledAppVersionHistoryRepository, - globalEnvVariables *util2.GlobalEnvVariables, helmAppService client2.HelmAppService, - manifestPushConfigRepository repository5.ManifestPushConfigRepository, - GitOpsManifestPushService GitOpsPushService, - argoClientWrapperService argocdServer.ArgoClientWrapperService, + globalEnvVariables *util2.GlobalEnvVariables, scopedVariableManager variables.ScopedVariableCMCSManager, - acdConfig *argocdServer.ACDConfig, -) *AppServiceImpl { + acdConfig *argocdServer.ACDConfig, chartRefService chartRef.ChartRefService) *AppServiceImpl { appServiceImpl := &AppServiceImpl{ environmentConfigRepository: environmentConfigRepository, mergeUtil: mergeUtil, pipelineOverrideRepository: pipelineOverrideRepository, logger: logger, - ciArtifactRepository: ciArtifactRepository, pipelineRepository: pipelineRepository, - dbMigrationConfigRepository: dbMigrationConfigRepository, eventClient: eventClient, eventFactory: eventFactory, acdClient: acdClient, - tokenCache: cache, - acdAuthConfig: authConfig, - enforcer: enforcer, - enforcerUtil: enforcerUtil, - user: user, - appListingRepository: appListingRepository, appRepository: appRepository, - envRepository: envRepository, - pipelineConfigRepository: pipelineConfigRepository, configMapRepository: configMapRepository, chartRepository: chartRepository, - ciPipelineMaterialRepository: ciPipelineMaterialRepository, cdWorkflowRepository: cdWorkflowRepository, commonService: commonService, - imageScanDeployInfoRepository: imageScanDeployInfoRepository, - imageScanHistoryRepository: imageScanHistoryRepository, - ArgoK8sClient: ArgoK8sClient, - gitFactory: gitFactory, - pipelineStrategyHistoryService: pipelineStrategyHistoryService, - configMapHistoryService: configMapHistoryService, - deploymentTemplateHistoryService: deploymentTemplateHistoryService, chartTemplateService: chartTemplateService, - refChartDir: refChartDir, - chartRefRepository: chartRefRepository, - chartService: chartService, - helmAppClient: helmAppClient, argoUserService: argoUserService, pipelineStatusTimelineRepository: cdPipelineStatusTimelineRepo, - appCrudOperationService: appCrudOperationService, - configMapHistoryRepository: configMapHistoryRepository, - strategyHistoryRepository: strategyHistoryRepository, - deploymentTemplateHistoryRepository: deploymentTemplateHistoryRepository, - dockerRegistryIpsConfigService: dockerRegistryIpsConfigService, pipelineStatusTimelineResourcesService: pipelineStatusTimelineResourcesService, pipelineStatusSyncDetailService: pipelineStatusSyncDetailService, pipelineStatusTimelineService: pipelineStatusTimelineService, appStatusConfig: appStatusConfig, - gitOpsConfigRepository: gitOpsConfigRepository, appStatusService: appStatusService, installedAppRepository: installedAppRepository, - AppStoreDeploymentService: AppStoreDeploymentService, - K8sCommonService: k8sCommonService, installedAppVersionHistoryRepository: installedAppVersionHistoryRepository, globalEnvVariables: globalEnvVariables, - helmAppService: helmAppService, - manifestPushConfigRepository: manifestPushConfigRepository, - GitOpsManifestPushService: GitOpsManifestPushService, - argoClientWrapperService: argoClientWrapperService, scopedVariableManager: scopedVariableManager, acdConfig: acdConfig, + chartRefService: chartRefService, } return appServiceImpl } @@ -984,16 +873,16 @@ func (impl *AppServiceImpl) BuildChartAndGetPath(appName string, envOverride *ch Name: appName, Version: envOverride.Chart.ChartVersion, } - referenceTemplatePath := path.Join(string(impl.refChartDir), envOverride.Chart.ReferenceTemplate) + referenceTemplatePath := path.Join(chartRepoRepository.RefChartDirPath, envOverride.Chart.ReferenceTemplate) // Load custom charts to referenceTemplatePath if not exists if _, err := os.Stat(referenceTemplatePath); os.IsNotExist(err) { - chartRefValue, err := impl.chartRefRepository.FindById(envOverride.Chart.ChartRefId) + chartRefValue, err := impl.chartRefService.FindById(envOverride.Chart.ChartRefId) if err != nil { impl.logger.Errorw("error in fetching ChartRef data", "err", err) return "", err } if chartRefValue.ChartData != nil { - chartInfo, err := impl.chartService.ExtractChartIfMissing(chartRefValue.ChartData, string(impl.refChartDir), chartRefValue.Location) + chartInfo, err := impl.chartRefService.ExtractChartIfMissing(chartRefValue.ChartData, chartRepoRepository.RefChartDirPath, chartRefValue.Location) if chartInfo != nil && chartInfo.TemporaryFolder != "" { err1 := os.RemoveAll(chartInfo.TemporaryFolder) if err1 != nil { @@ -1065,7 +954,7 @@ func (impl *AppServiceImpl) autoHealChartLocationInChart(ctx context.Context, en // get chart ref from DB (to get location) chartRefId := chart.ChartRefId _, span = otel.Tracer("orchestrator").Start(ctx, "chartRefRepository.FindById") - chartRef, err := impl.chartRefRepository.FindById(chartRefId) + chartRefDto, err := impl.chartRefService.FindById(chartRefId) span.End() if err != nil { impl.logger.Errorw("error occurred while fetching chartRef from DB", "chartRefId", chartRefId, "err", err) @@ -1073,7 +962,7 @@ func (impl *AppServiceImpl) autoHealChartLocationInChart(ctx context.Context, en } // build new chart location - newChartLocation := filepath.Join(chartRef.Location, envOverride.Chart.ChartVersion) + newChartLocation := filepath.Join(chartRefDto.Location, envOverride.Chart.ChartVersion) impl.logger.Infow("new chart location build", "chartId", chartId, "newChartLocation", newChartLocation) // update chart in DB diff --git a/pkg/app/ManifestPushService.go b/pkg/app/ManifestPushService.go index 7f370435e4..f52731b204 100644 --- a/pkg/app/ManifestPushService.go +++ b/pkg/app/ManifestPushService.go @@ -11,7 +11,7 @@ import ( . "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app/bean" status2 "github.com/devtron-labs/devtron/pkg/app/status" - chartService "github.com/devtron-labs/devtron/pkg/chart" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" "github.com/go-pg/pg" "go.opentelemetry.io/otel" "go.uber.org/zap" @@ -29,33 +29,31 @@ type GitOpsPushService interface { type GitOpsManifestPushServiceImpl struct { logger *zap.SugaredLogger chartTemplateService util.ChartTemplateService - chartService chartService.ChartService gitOpsConfigRepository repository.GitOpsConfigRepository gitFactory *GitFactory pipelineStatusTimelineService status2.PipelineStatusTimelineService pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository acdConfig *argocdServer.ACDConfig + chartRefService chartRef.ChartRefService } func NewGitOpsManifestPushServiceImpl( logger *zap.SugaredLogger, chartTemplateService util.ChartTemplateService, - chartService chartService.ChartService, gitOpsConfigRepository repository.GitOpsConfigRepository, gitFactory *GitFactory, pipelineStatusTimelineService status2.PipelineStatusTimelineService, pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository, - acdConfig *argocdServer.ACDConfig, -) *GitOpsManifestPushServiceImpl { + acdConfig *argocdServer.ACDConfig, chartRefService chartRef.ChartRefService) *GitOpsManifestPushServiceImpl { return &GitOpsManifestPushServiceImpl{ logger: logger, chartTemplateService: chartTemplateService, - chartService: chartService, gitOpsConfigRepository: gitOpsConfigRepository, gitFactory: gitFactory, pipelineStatusTimelineService: pipelineStatusTimelineService, pipelineStatusTimelineRepository: pipelineStatusTimelineRepository, acdConfig: acdConfig, + chartRefService: chartRefService, } } @@ -110,7 +108,7 @@ func (impl *GitOpsManifestPushServiceImpl) PushChartToGitRepo(manifestPushTempla gitOpsRepoName := impl.chartTemplateService.GetGitOpsRepoName(manifestPushTemplate.AppName) span.End() _, span = otel.Tracer("orchestrator").Start(ctx, "chartService.CheckChartExists") - err := impl.chartService.CheckChartExists(manifestPushTemplate.ChartRefId) + err := impl.chartRefService.CheckChartExists(manifestPushTemplate.ChartRefId) span.End() if err != nil { impl.logger.Errorw("err in getting chart info", "err", err) diff --git a/pkg/app/integrationTest/AppService_test.go b/pkg/app/integrationTest/AppService_test.go index 9c78b5b783..03ed6b6423 100644 --- a/pkg/app/integrationTest/AppService_test.go +++ b/pkg/app/integrationTest/AppService_test.go @@ -155,12 +155,11 @@ func InitAppService() *app2.AppServiceImpl { pipelineStatusSyncDetailRepository := pipelineConfig.NewPipelineStatusSyncDetailRepositoryImpl(dbConnection, logger) pipelineStatusSyncDetailService := status.NewPipelineStatusSyncDetailServiceImpl(logger, pipelineStatusSyncDetailRepository) pipelineStatusTimelineService := status.NewPipelineStatusTimelineServiceImpl(logger, pipelineStatusTimelineRepository, cdWorkflowRepository, nil, pipelineStatusTimelineResourcesService, pipelineStatusSyncDetailService, nil, nil) - refChartDir := chartRepoRepository.RefChartDir("scripts/devtron-reference-helm-charts") appService := app2.NewAppService(nil, pipelineOverrideRepository, nil, logger, nil, pipelineRepository, nil, eventClient, eventFactory, nil, nil, nil, nil, nil, nil, appListingRepository, appRepository, nil, nil, nil, nil, nil, chartRepository, nil, cdWorkflowRepository, nil, nil, nil, nil, - nil, nil, nil, nil, nil, refChartDir, nil, + nil, nil, nil, nil, nil, nil, nil, nil, nil, pipelineStatusTimelineRepository, nil, nil, nil, nil, nil, pipelineStatusTimelineResourcesService, pipelineStatusSyncDetailService, pipelineStatusTimelineService, nil, nil, nil, nil, nil, nil, nil, diff --git a/pkg/appStore/bean/bean.go b/pkg/appStore/bean/bean.go index 19dde14013..e9763577f3 100644 --- a/pkg/appStore/bean/bean.go +++ b/pkg/appStore/bean/bean.go @@ -150,6 +150,10 @@ type ChartGroupInstallAppRes struct { // / type RefChartProxyDir string +const ( + RefChartProxyDirPath = "scripts/devtron-reference-helm-charts" +) + var CHART_PROXY_TEMPLATE = "reference-chart-proxy" var REQUIREMENTS_YAML_FILE = "requirements.yaml" var VALUES_YAML_FILE = "values.yaml" diff --git a/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go b/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go index 44ca59836e..ab912fda04 100644 --- a/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go +++ b/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go @@ -66,7 +66,6 @@ type AppStoreDeploymentCommonServiceImpl struct { appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository environmentRepository repository2.EnvironmentRepository chartTemplateService util.ChartTemplateService - refChartDir appStoreBean.RefChartProxyDir gitFactory *util.GitFactory gitOpsConfigRepository repository3.GitOpsConfigRepository } @@ -77,7 +76,6 @@ func NewAppStoreDeploymentCommonServiceImpl( appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository, environmentRepository repository2.EnvironmentRepository, chartTemplateService util.ChartTemplateService, - refChartDir appStoreBean.RefChartProxyDir, gitFactory *util.GitFactory, gitOpsConfigRepository repository3.GitOpsConfigRepository, ) *AppStoreDeploymentCommonServiceImpl { @@ -87,7 +85,6 @@ func NewAppStoreDeploymentCommonServiceImpl( appStoreApplicationVersionRepository: appStoreApplicationVersionRepository, environmentRepository: environmentRepository, chartTemplateService: chartTemplateService, - refChartDir: refChartDir, gitFactory: gitFactory, gitOpsConfigRepository: gitOpsConfigRepository, } @@ -352,7 +349,7 @@ func (impl AppStoreDeploymentCommonServiceImpl) CreateChartProxyAndGetPath(insta ChartCreateResponse := &util.ChartCreateResponse{} template := appStoreBean.CHART_PROXY_TEMPLATE - chartPath := path.Join(string(impl.refChartDir), template) + chartPath := path.Join(appStoreBean.RefChartProxyDirPath, template) valid, err := chartutil.IsChartDir(chartPath) if err != nil || !valid { impl.logger.Errorw("invalid base chart", "dir", chartPath, "err", err) diff --git a/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go b/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go index 179c59463b..77d651bf19 100644 --- a/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go +++ b/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go @@ -72,7 +72,6 @@ type AppStoreDeploymentFullModeService interface { type AppStoreDeploymentFullModeServiceImpl struct { logger *zap.SugaredLogger chartTemplateService util.ChartTemplateService - refChartDir appStoreBean.RefChartProxyDir repositoryService repository.ServiceClient appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository environmentRepository repository5.EnvironmentRepository @@ -94,7 +93,7 @@ type AppStoreDeploymentFullModeServiceImpl struct { } func NewAppStoreDeploymentFullModeServiceImpl(logger *zap.SugaredLogger, - chartTemplateService util.ChartTemplateService, refChartDir appStoreBean.RefChartProxyDir, + chartTemplateService util.ChartTemplateService, repositoryService repository.ServiceClient, appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository, environmentRepository repository5.EnvironmentRepository, @@ -114,7 +113,6 @@ func NewAppStoreDeploymentFullModeServiceImpl(logger *zap.SugaredLogger, appStoreDeploymentFullModeServiceImpl := &AppStoreDeploymentFullModeServiceImpl{ logger: logger, chartTemplateService: chartTemplateService, - refChartDir: refChartDir, repositoryService: repositoryService, appStoreApplicationVersionRepository: appStoreApplicationVersionRepository, environmentRepository: environmentRepository, @@ -156,7 +154,7 @@ func (impl AppStoreDeploymentFullModeServiceImpl) AppStoreDeployOperationGIT(ins //STEP 1: Commit and PUSH on Gitlab template := appStoreBean.CHART_PROXY_TEMPLATE - chartPath := path.Join(string(impl.refChartDir), template) + chartPath := path.Join(appStoreBean.RefChartProxyDirPath, template) valid, err := chartutil.IsChartDir(chartPath) if err != nil || !valid { impl.logger.Errorw("invalid base chart", "dir", chartPath, "err", err) diff --git a/pkg/appStore/deployment/service/InstalledAppService.go b/pkg/appStore/deployment/service/InstalledAppService.go index 47af5f5dc3..f97b19e5eb 100644 --- a/pkg/appStore/deployment/service/InstalledAppService.go +++ b/pkg/appStore/deployment/service/InstalledAppService.go @@ -52,7 +52,6 @@ import ( appStoreDeploymentCommon "github.com/devtron-labs/devtron/pkg/appStore/deployment/common" appStoreDeploymentFullMode "github.com/devtron-labs/devtron/pkg/appStore/deployment/fullMode" repository2 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" - appStoreDeploymentGitopsTool "github.com/devtron-labs/devtron/pkg/appStore/deployment/tool/gitops" appStoreDiscoverRepository "github.com/devtron-labs/devtron/pkg/appStore/discover/repository" "github.com/devtron-labs/devtron/pkg/appStore/values/service" "github.com/devtron-labs/devtron/pkg/auth/user" @@ -109,7 +108,6 @@ type InstalledAppServiceImpl struct { logger *zap.SugaredLogger installedAppRepository repository2.InstalledAppRepository chartTemplateService util.ChartTemplateService - refChartDir appStoreBean.RefChartProxyDir repositoryService repository.ServiceClient appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository environmentRepository repository5.EnvironmentRepository @@ -124,15 +122,14 @@ type InstalledAppServiceImpl struct { ArgoK8sClient argocdServer.ArgoK8sClient gitFactory *util.GitFactory aCDAuthConfig *util2.ACDAuthConfig - gitOpsRepository repository3.GitOpsConfigRepository - userService user.UserService - appStoreDeploymentService AppStoreDeploymentService + gitOpsRepository repository3.GitOpsConfigRepository + userService user.UserService + appStoreDeploymentService AppStoreDeploymentService appStoreDeploymentFullModeService appStoreDeploymentFullMode.AppStoreDeploymentFullModeService installedAppRepositoryHistory repository2.InstalledAppVersionHistoryRepository argoUserService argo.ArgoUserService helmAppClient client.HelmAppClient helmAppService client.HelmAppService - attributesRepository repository3.AttributesRepository appStatusService appStatus.AppStatusService K8sUtil *util4.K8sUtil pipelineStatusTimelineService status.PipelineStatusTimelineService @@ -144,7 +141,7 @@ type InstalledAppServiceImpl struct { func NewInstalledAppServiceImpl(logger *zap.SugaredLogger, installedAppRepository repository2.InstalledAppRepository, - chartTemplateService util.ChartTemplateService, refChartDir appStoreBean.RefChartProxyDir, + chartTemplateService util.ChartTemplateService, repositoryService repository.ServiceClient, appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository, environmentRepository repository5.EnvironmentRepository, teamRepository repository4.TeamRepository, @@ -160,17 +157,15 @@ func NewInstalledAppServiceImpl(logger *zap.SugaredLogger, appStoreDeploymentService AppStoreDeploymentService, installedAppRepositoryHistory repository2.InstalledAppVersionHistoryRepository, argoUserService argo.ArgoUserService, helmAppClient client.HelmAppClient, helmAppService client.HelmAppService, - attributesRepository repository3.AttributesRepository, appStatusService appStatus.AppStatusService, K8sUtil *util4.K8sUtil, pipelineStatusTimelineService status.PipelineStatusTimelineService, appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService, - appStoreDeploymentArgoCdService appStoreDeploymentGitopsTool.AppStoreDeploymentArgoCdService, k8sCommonService k8s.K8sCommonService, k8sApplicationService application3.K8sApplicationService, + k8sCommonService k8s.K8sCommonService, k8sApplicationService application3.K8sApplicationService, acdConfig *argocdServer.ACDConfig) (*InstalledAppServiceImpl, error) { impl := &InstalledAppServiceImpl{ logger: logger, installedAppRepository: installedAppRepository, chartTemplateService: chartTemplateService, - refChartDir: refChartDir, repositoryService: repositoryService, appStoreApplicationVersionRepository: appStoreApplicationVersionRepository, environmentRepository: environmentRepository, @@ -193,7 +188,6 @@ func NewInstalledAppServiceImpl(logger *zap.SugaredLogger, argoUserService: argoUserService, helmAppClient: helmAppClient, helmAppService: helmAppService, - attributesRepository: attributesRepository, appStatusService: appStatusService, K8sUtil: K8sUtil, pipelineStatusTimelineService: pipelineStatusTimelineService, diff --git a/pkg/appStore/deployment/service/InstalledAppService_test.go b/pkg/appStore/deployment/service/InstalledAppService_test.go index 3f8bb5d4de..13743864df 100644 --- a/pkg/appStore/deployment/service/InstalledAppService_test.go +++ b/pkg/appStore/deployment/service/InstalledAppService_test.go @@ -10,7 +10,6 @@ import ( repository3 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/util" - appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" appStoreDeploymentFullMode "github.com/devtron-labs/devtron/pkg/appStore/deployment/fullMode" repository4 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" appStoreDiscoverRepository "github.com/devtron-labs/devtron/pkg/appStore/discover/repository" @@ -28,7 +27,6 @@ func TestInstalledAppServiceImpl_DeployDefaultChartOnCluster(t *testing.T) { logger *zap.SugaredLogger installedAppRepository repository4.InstalledAppRepository chartTemplateService util.ChartTemplateService - refChartDir appStoreBean.RefChartProxyDir repositoryService repository2.ServiceClient appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository environmentRepository repository.EnvironmentRepository @@ -43,9 +41,9 @@ func TestInstalledAppServiceImpl_DeployDefaultChartOnCluster(t *testing.T) { ArgoK8sClient argocdServer.ArgoK8sClient gitFactory *util.GitFactory aCDAuthConfig *util2.ACDAuthConfig - gitOpsRepository repository3.GitOpsConfigRepository - userService user.UserService - appStoreDeploymentService AppStoreDeploymentService + gitOpsRepository repository3.GitOpsConfigRepository + userService user.UserService + appStoreDeploymentService AppStoreDeploymentService appStoreDeploymentFullModeService appStoreDeploymentFullMode.AppStoreDeploymentFullModeService } type args struct { @@ -67,7 +65,6 @@ func TestInstalledAppServiceImpl_DeployDefaultChartOnCluster(t *testing.T) { logger: tt.fields.logger, installedAppRepository: tt.fields.installedAppRepository, chartTemplateService: tt.fields.chartTemplateService, - refChartDir: tt.fields.refChartDir, repositoryService: tt.fields.repositoryService, appStoreApplicationVersionRepository: tt.fields.appStoreApplicationVersionRepository, environmentRepository: tt.fields.environmentRepository, diff --git a/pkg/bulkAction/BulkUpdateService.go b/pkg/bulkAction/BulkUpdateService.go index 9991ca32ee..4ae831d949 100644 --- a/pkg/bulkAction/BulkUpdateService.go +++ b/pkg/bulkAction/BulkUpdateService.go @@ -12,7 +12,6 @@ import ( client "github.com/devtron-labs/devtron/api/helm-app" openapi "github.com/devtron-labs/devtron/api/helm-app/openapiClient" "github.com/devtron-labs/devtron/client/argocdServer/application" - "github.com/devtron-labs/devtron/client/argocdServer/repository" "github.com/devtron-labs/devtron/internal/sql/models" "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/appWorkflow" @@ -22,10 +21,11 @@ import ( "github.com/devtron-labs/devtron/internal/util" appWorkflow2 "github.com/devtron-labs/devtron/pkg/appWorkflow" bean2 "github.com/devtron-labs/devtron/pkg/bean" - "github.com/devtron-labs/devtron/pkg/chart" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" + bean3 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/bean" "github.com/devtron-labs/devtron/pkg/pipeline" pipeline1 "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/history" @@ -66,27 +66,14 @@ type BulkUpdateServiceImpl struct { bulkUpdateRepository bulkUpdate.BulkUpdateRepository chartRepository chartRepoRepository.ChartRepository logger *zap.SugaredLogger - repoRepository chartRepoRepository.ChartRepoRepository - chartTemplateService util.ChartTemplateService - mergeUtil util.MergeUtil - repositoryService repository.ServiceClient - defaultChart chart.DefaultChart - chartRefRepository chartRepoRepository.ChartRefRepository - envOverrideRepository chartConfig.EnvConfigOverrideRepository - pipelineConfigRepository chartConfig.PipelineConfigRepository - configMapRepository chartConfig.ConfigMapRepository environmentRepository repository2.EnvironmentRepository pipelineRepository pipelineConfig.PipelineRepository - client *http.Client appRepository app.AppRepository deploymentTemplateHistoryService history.DeploymentTemplateHistoryService configMapHistoryService history.ConfigMapHistoryService workflowDagExecutor pipeline.WorkflowDagExecutor - cdWorkflowRepository pipelineConfig.CdWorkflowRepository pipelineBuilder pipeline.PipelineBuilder - helmAppService client.HelmAppService enforcerUtil rbac.EnforcerUtil - enforcerUtilHelm rbac.EnforcerUtilHelm ciHandler pipeline.CiHandler ciPipelineRepository pipelineConfig.CiPipelineRepository appWorkflowRepository appWorkflow.AppWorkflowRepository @@ -95,61 +82,40 @@ type BulkUpdateServiceImpl struct { argoUserService argo.ArgoUserService scopedVariableManager variables.ScopedVariableManager deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService + chartRefService chartRef.ChartRefService } func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateRepository, chartRepository chartRepoRepository.ChartRepository, logger *zap.SugaredLogger, - chartTemplateService util.ChartTemplateService, - repoRepository chartRepoRepository.ChartRepoRepository, - defaultChart chart.DefaultChart, - mergeUtil util.MergeUtil, - repositoryService repository.ServiceClient, - chartRefRepository chartRepoRepository.ChartRefRepository, - envOverrideRepository chartConfig.EnvConfigOverrideRepository, - pipelineConfigRepository chartConfig.PipelineConfigRepository, - configMapRepository chartConfig.ConfigMapRepository, environmentRepository repository2.EnvironmentRepository, pipelineRepository pipelineConfig.PipelineRepository, - client *http.Client, appRepository app.AppRepository, deploymentTemplateHistoryService history.DeploymentTemplateHistoryService, configMapHistoryService history.ConfigMapHistoryService, workflowDagExecutor pipeline.WorkflowDagExecutor, - cdWorkflowRepository pipelineConfig.CdWorkflowRepository, pipelineBuilder pipeline.PipelineBuilder, - helmAppService client.HelmAppService, enforcerUtil rbac.EnforcerUtil, - enforcerUtilHelm rbac.EnforcerUtilHelm, ciHandler pipeline.CiHandler, + pipelineBuilder pipeline.PipelineBuilder, + enforcerUtil rbac.EnforcerUtil, + ciHandler pipeline.CiHandler, ciPipelineRepository pipelineConfig.CiPipelineRepository, appWorkflowRepository appWorkflow.AppWorkflowRepository, appWorkflowService appWorkflow2.AppWorkflowService, pubsubClient *pubsub.PubSubClientServiceImpl, argoUserService argo.ArgoUserService, scopedVariableManager variables.ScopedVariableManager, - deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService) (*BulkUpdateServiceImpl, error) { + deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService, + chartRefService chartRef.ChartRefService) (*BulkUpdateServiceImpl, error) { impl := &BulkUpdateServiceImpl{ bulkUpdateRepository: bulkUpdateRepository, chartRepository: chartRepository, logger: logger, - chartTemplateService: chartTemplateService, - repoRepository: repoRepository, - mergeUtil: mergeUtil, - defaultChart: defaultChart, - repositoryService: repositoryService, - chartRefRepository: chartRefRepository, - envOverrideRepository: envOverrideRepository, - pipelineConfigRepository: pipelineConfigRepository, - configMapRepository: configMapRepository, environmentRepository: environmentRepository, pipelineRepository: pipelineRepository, - client: client, appRepository: appRepository, deploymentTemplateHistoryService: deploymentTemplateHistoryService, configMapHistoryService: configMapHistoryService, workflowDagExecutor: workflowDagExecutor, - cdWorkflowRepository: cdWorkflowRepository, pipelineBuilder: pipelineBuilder, - helmAppService: helmAppService, enforcerUtil: enforcerUtil, - enforcerUtilHelm: enforcerUtilHelm, ciHandler: ciHandler, ciPipelineRepository: ciPipelineRepository, appWorkflowRepository: appWorkflowRepository, @@ -158,6 +124,7 @@ func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateReposito argoUserService: argoUserService, scopedVariableManager: scopedVariableManager, deployedAppMetricsService: deployedAppMetricsService, + chartRefService: chartRefService, } err := impl.SubscribeToCdBulkTriggerTopic() @@ -1099,14 +1066,14 @@ func (impl BulkUpdateServiceImpl) buildHibernateUnHibernateRequestForHelmPipelin } hibernateRequest := &openapi.HibernateRequest{} - chartInfo, err := impl.chartRefRepository.FetchInfoOfChartConfiguredInApp(pipeline.AppId) + chartInfo, err := impl.chartRefService.FetchInfoOfChartConfiguredInApp(pipeline.AppId) if err != nil { impl.logger.Errorw("error in getting chart info for chart configured in app", "err", err, "appId", pipeline.AppId) return nil, nil, err } var group, kind, version, name string name = fmt.Sprintf("%s-%s", pipeline.App.AppName, pipeline.Environment.Name) - if chartInfo.Name == "" && chartInfo.UserUploaded == false { + if chartInfo.Name == bean3.RolloutChartType && chartInfo.UserUploaded == false { // rollout type chart group = "argoproj.io" kind = "Rollout" @@ -1122,7 +1089,7 @@ func (impl BulkUpdateServiceImpl) buildHibernateUnHibernateRequestForHelmPipelin }, }, } - } else if chartInfo.Name == "Deployment" { + } else if chartInfo.Name == bean3.DeploymentChartType { //deployment type chart group = "apps" kind = "Deployment" diff --git a/pkg/chart/ChartCompatibility_test.go b/pkg/chart/ChartCompatibility_test.go index 95005d1916..5bafe12b0f 100644 --- a/pkg/chart/ChartCompatibility_test.go +++ b/pkg/chart/ChartCompatibility_test.go @@ -1,6 +1,8 @@ package chart import ( + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/bean" "reflect" "testing" ) @@ -18,8 +20,8 @@ func Test_checkCompatibility(t *testing.T) { { name: "when charts are compatible, it should return true", args: args{ - oldChartType: DeploymentChartType, - newChartType: RolloutChartType, + oldChartType: bean.DeploymentChartType, + newChartType: bean.RolloutChartType, }, want: true, }, @@ -27,14 +29,14 @@ func Test_checkCompatibility(t *testing.T) { name: "when oldChart is not found, it should return false", args: args{ oldChartType: "Sdfasdf", - newChartType: RolloutChartType, + newChartType: bean.RolloutChartType, }, want: false, }, { name: "when newChart is not found, it should return false", args: args{ - oldChartType: DeploymentChartType, + oldChartType: bean.DeploymentChartType, newChartType: "random sdfasdf saldfj", }, want: false, @@ -42,7 +44,7 @@ func Test_checkCompatibility(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := CheckCompatibility(tt.args.oldChartType, tt.args.newChartType); got != tt.want { + if got := chartRef.CheckCompatibility(tt.args.oldChartType, tt.args.newChartType); got != tt.want { t.Errorf("checkCompatibility() = %v, want %v", got, tt.want) } }) @@ -68,14 +70,14 @@ func TestCompatibleChartsWith(t *testing.T) { { name: "when chart is found, it should return a slice of all cpmpatible chartIds", args: args{ - chartType: DeploymentChartType, + chartType: bean.DeploymentChartType, }, - want: []string{RolloutChartType}, + want: []string{bean.RolloutChartType}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := CompatibleChartsWith(tt.args.chartType); !reflect.DeepEqual(got, tt.want) { + if got := chartRef.CompatibleChartsWith(tt.args.chartType); !reflect.DeepEqual(got, tt.want) { t.Errorf("CompatibleChartsWith() = %v, want %v", got, tt.want) } }) diff --git a/pkg/chart/ChartService.go b/pkg/chart/ChartService.go index ceb5956051..0212bc570a 100644 --- a/pkg/chart/ChartService.go +++ b/pkg/chart/ChartService.go @@ -24,6 +24,8 @@ import ( "fmt" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/bean" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" + bean2 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/bean" "github.com/devtron-labs/devtron/pkg/resourceQualifiers" "github.com/devtron-labs/devtron/pkg/variables" "github.com/devtron-labs/devtron/pkg/variables/parsers" @@ -31,8 +33,6 @@ import ( "go.opentelemetry.io/otel" - "github.com/devtron-labs/devtron/internal/constants" - //"github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/internal/sql/repository/app" @@ -40,29 +40,22 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline/history" "io/ioutil" - "net/http" "os" - "path" "path/filepath" "strconv" "strings" "time" - repository4 "github.com/devtron-labs/devtron/pkg/cluster/repository" - "github.com/devtron-labs/devtron/pkg/sql" - dirCopy "github.com/otiai10/copy" - - "github.com/devtron-labs/devtron/client/argocdServer/repository" "github.com/devtron-labs/devtron/internal/sql/models" "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" + repository4 "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/sql" util2 "github.com/devtron-labs/devtron/util" "github.com/go-pg/pg" "github.com/juju/errors" "github.com/xeipuuv/gojsonschema" "go.uber.org/zap" - "k8s.io/helm/pkg/chartutil" "k8s.io/helm/pkg/proto/hapi/chart" "sigs.k8s.io/yaml" ) @@ -75,28 +68,17 @@ type ChartService interface { GetAppOverrideForDefaultTemplate(chartRefId int) (map[string]interface{}, string, error) UpdateAppOverride(ctx context.Context, templateRequest *TemplateRequest) (*TemplateRequest, error) IsReadyToTrigger(appId int, envId int, pipelineId int) (IsReady, error) - ChartRefAutocomplete() ([]ChartRef, error) - ChartRefAutocompleteForAppOrEnv(appId int, envId int) (*ChartRefResponse, error) FindPreviousChartByAppId(appId int) (chartTemplate *TemplateRequest, err error) UpgradeForApp(appId int, chartRefId int, newAppOverride map[string]interface{}, userId int32, ctx context.Context) (bool, error) DeploymentTemplateValidate(ctx context.Context, templatejson interface{}, chartRefId int, scope resourceQualifiers.Scope) (bool, error) JsonSchemaExtractFromFile(chartRefId int) (map[string]interface{}, string, error) GetSchemaAndReadmeForTemplateByChartRefId(chartRefId int) (schema []byte, readme []byte, err error) - ExtractChartIfMissing(chartData []byte, refChartDir string, location string) (*ChartDataInfo, error) - CheckChartExists(chartRefId int) error - CheckIsAppMetricsSupported(chartRefId int) (bool, error) - GetLocationFromChartNameAndVersion(chartName string, chartVersion string) string - FormatChartName(chartName string) string ValidateUploadedFileFormat(fileName string) error - ReadChartMetaDataForLocation(chartDir string, fileName string) (*ChartYamlStruct, error) - FetchCustomChartsInfo() ([]*ChartDto, error) - CheckCustomChartByAppId(id int) (bool, error) - CheckCustomChartByChartId(id int) (bool, error) - ChartRefIdsCompatible(oldChartRefId int, newChartRefId int) (bool, string, string) + CheckIfChartRefUserUploadedByAppId(id int) (bool, error) PatchEnvOverrides(values json.RawMessage, oldChartType string, newChartType string) (json.RawMessage, error) FlaggerCanaryEnabled(values json.RawMessage) (bool, error) - GetCustomChartInBytes(chatRefId int) ([]byte, error) - GetRefChart(templateRequest TemplateRequest) (string, string, error, string, string) + + ChartRefAutocompleteForAppOrEnv(appId int, envId int) (*bean2.ChartRefAutocompleteResponse, error) } type ChartServiceImpl struct { @@ -106,19 +88,13 @@ type ChartServiceImpl struct { chartTemplateService util.ChartTemplateService pipelineGroupRepository app.AppRepository mergeUtil util.MergeUtil - repositoryService repository.ServiceClient - refChartDir chartRepoRepository.RefChartDir - defaultChart DefaultChart - chartRefRepository chartRepoRepository.ChartRefRepository envOverrideRepository chartConfig.EnvConfigOverrideRepository pipelineConfigRepository chartConfig.PipelineConfigRepository - configMapRepository chartConfig.ConfigMapRepository environmentRepository repository4.EnvironmentRepository - pipelineRepository pipelineConfig.PipelineRepository - client *http.Client deploymentTemplateHistoryService history.DeploymentTemplateHistoryService scopedVariableManager variables.ScopedVariableManager deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService + chartRefService chartRef.ChartRefService } func NewChartServiceImpl(chartRepository chartRepoRepository.ChartRepository, @@ -126,24 +102,14 @@ func NewChartServiceImpl(chartRepository chartRepoRepository.ChartRepository, chartTemplateService util.ChartTemplateService, repoRepository chartRepoRepository.ChartRepoRepository, pipelineGroupRepository app.AppRepository, - refChartDir chartRepoRepository.RefChartDir, - defaultChart DefaultChart, mergeUtil util.MergeUtil, - repositoryService repository.ServiceClient, - chartRefRepository chartRepoRepository.ChartRefRepository, envOverrideRepository chartConfig.EnvConfigOverrideRepository, pipelineConfigRepository chartConfig.PipelineConfigRepository, - configMapRepository chartConfig.ConfigMapRepository, environmentRepository repository4.EnvironmentRepository, - pipelineRepository pipelineConfig.PipelineRepository, - client *http.Client, deploymentTemplateHistoryService history.DeploymentTemplateHistoryService, scopedVariableManager variables.ScopedVariableManager, - deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService) *ChartServiceImpl { - - // cache devtron reference charts list - devtronChartList, _ := chartRefRepository.FetchAllChartInfoByUploadFlag(false) - SetReservedChartList(devtronChartList) + deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService, + chartRefService chartRef.ChartRefService) *ChartServiceImpl { return &ChartServiceImpl{ chartRepository: chartRepository, logger: logger, @@ -151,40 +117,16 @@ func NewChartServiceImpl(chartRepository chartRepoRepository.ChartRepository, repoRepository: repoRepository, pipelineGroupRepository: pipelineGroupRepository, mergeUtil: mergeUtil, - refChartDir: refChartDir, - defaultChart: defaultChart, - repositoryService: repositoryService, - chartRefRepository: chartRefRepository, envOverrideRepository: envOverrideRepository, pipelineConfigRepository: pipelineConfigRepository, - configMapRepository: configMapRepository, environmentRepository: environmentRepository, - pipelineRepository: pipelineRepository, - client: client, deploymentTemplateHistoryService: deploymentTemplateHistoryService, scopedVariableManager: scopedVariableManager, deployedAppMetricsService: deployedAppMetricsService, + chartRefService: chartRefService, } } -func (impl ChartServiceImpl) ChartRefIdsCompatible(oldChartRefId int, newChartRefId int) (bool, string, string) { - oldChart, err := impl.chartRefRepository.FindById(oldChartRefId) - if err != nil { - return false, "", "" - } - newChart, err := impl.chartRefRepository.FindById(newChartRefId) - if err != nil { - return false, "", "" - } - if len(oldChart.Name) == 0 { - oldChart.Name = RolloutChartType - } - if len(newChart.Name) == 0 { - newChart.Name = RolloutChartType - } - return CheckCompatibility(oldChart.Name, newChart.Name), oldChart.Name, newChart.Name -} - func (impl ChartServiceImpl) FlaggerCanaryEnabled(values json.RawMessage) (bool, error) { var jsonMap map[string]json.RawMessage if err := json.Unmarshal([]byte(values), &jsonMap); err != nil { @@ -210,14 +152,14 @@ func (impl ChartServiceImpl) PatchEnvOverrides(values json.RawMessage, oldChartT } func (impl ChartServiceImpl) GetSchemaAndReadmeForTemplateByChartRefId(chartRefId int) ([]byte, []byte, error) { - refChart, _, err, _, _ := impl.GetRefChart(TemplateRequest{ChartRefId: chartRefId}) + refChart, _, err, _, _ := impl.chartRefService.GetRefChart(chartRefId) if err != nil { impl.logger.Errorw("error in getting refChart", "err", err, "chartRefId", chartRefId) return nil, nil, err } var schemaByte []byte var readmeByte []byte - err = impl.CheckChartExists(chartRefId) + err = impl.chartRefService.CheckChartExists(chartRefId) if err != nil { impl.logger.Errorw("error in getting refChart", "err", err, "chartRefId", chartRefId) return nil, nil, err @@ -234,13 +176,13 @@ func (impl ChartServiceImpl) GetSchemaAndReadmeForTemplateByChartRefId(chartRefI } func (impl ChartServiceImpl) GetAppOverrideForDefaultTemplate(chartRefId int) (map[string]interface{}, string, error) { - err := impl.CheckChartExists(chartRefId) + err := impl.chartRefService.CheckChartExists(chartRefId) if err != nil { impl.logger.Errorw("error in getting missing chart for chartRefId", "err", err, "chartRefId") return nil, "", err } - refChart, _, err, _, _ := impl.GetRefChart(TemplateRequest{ChartRefId: chartRefId}) + refChart, _, err, _, _ := impl.chartRefService.GetRefChart(chartRefId) if err != nil { return nil, "", err } @@ -292,7 +234,7 @@ type AppMetricsEnabled struct { } func (impl ChartServiceImpl) Create(templateRequest TemplateRequest, ctx context.Context) (*TemplateRequest, error) { - err := impl.CheckChartExists(templateRequest.ChartRefId) + err := impl.chartRefService.CheckChartExists(templateRequest.ChartRefId) if err != nil { impl.logger.Errorw("error in getting missing chart for chartRefId", "err", err, "chartRefId") return nil, err @@ -310,7 +252,7 @@ func (impl ChartServiceImpl) Create(templateRequest TemplateRequest, ctx context return nil, err } - refChart, templateName, err, _, pipelineStrategyPath := impl.GetRefChart(templateRequest) + refChart, templateName, err, _, pipelineStrategyPath := impl.chartRefService.GetRefChart(templateRequest.ChartRefId) if err != nil { return nil, err } @@ -455,7 +397,7 @@ func (impl ChartServiceImpl) Create(templateRequest TemplateRequest, ctx context } func (impl ChartServiceImpl) CreateChartFromEnvOverride(templateRequest TemplateRequest, ctx context.Context) (*TemplateRequest, error) { - err := impl.CheckChartExists(templateRequest.ChartRefId) + err := impl.chartRefService.CheckChartExists(templateRequest.ChartRefId) if err != nil { impl.logger.Errorw("error in getting missing chart for chartRefId", "err", err, "chartRefId") return nil, err @@ -476,7 +418,7 @@ func (impl ChartServiceImpl) CreateChartFromEnvOverride(templateRequest Template return nil, err } - refChart, templateName, err, _, pipelineStrategyPath := impl.GetRefChart(templateRequest) + refChart, templateName, err, _, pipelineStrategyPath := impl.chartRefService.GetRefChart(templateRequest.ChartRefId) if err != nil { return nil, err } @@ -600,75 +542,24 @@ func (impl ChartServiceImpl) getChartMetaData(templateRequest TemplateRequest) ( } return metadata, err } -func (impl ChartServiceImpl) GetRefChart(templateRequest TemplateRequest) (string, string, error, string, string) { - var template string - var version string - //path of file in chart from where strategy config is to be taken - var pipelineStrategyPath string - if templateRequest.ChartRefId > 0 { - chartRef, err := impl.chartRefRepository.FindById(templateRequest.ChartRefId) - if err != nil { - chartRef, err = impl.chartRefRepository.GetDefault() - if err != nil { - return "", "", err, "", "" - } - } else if chartRef.UserUploaded { - refChartLocation := filepath.Join(string(impl.refChartDir), chartRef.Location) - if _, err := os.Stat(refChartLocation); os.IsNotExist(err) { - chartInfo, err := impl.ExtractChartIfMissing(chartRef.ChartData, string(impl.refChartDir), chartRef.Location) - if chartInfo != nil && chartInfo.TemporaryFolder != "" { - err1 := os.RemoveAll(chartInfo.TemporaryFolder) - if err1 != nil { - impl.logger.Errorw("error in deleting temp dir ", "err", err) - } - } - if err != nil { - impl.logger.Errorw("Error regarding uploaded chart", "err", err) - return "", "", err, "", "" - } - - } - } - template = chartRef.Location - version = chartRef.Version - pipelineStrategyPath = chartRef.DeploymentStrategyPath - } else { - chartRef, err := impl.chartRefRepository.GetDefault() - if err != nil { - return "", "", err, "", "" - } - template = chartRef.Location - version = chartRef.Version - pipelineStrategyPath = chartRef.DeploymentStrategyPath - } - - //TODO VIKI- fetch from chart ref table - chartPath := path.Join(string(impl.refChartDir), template) - valid, err := chartutil.IsChartDir(chartPath) - if err != nil || !valid { - impl.logger.Errorw("invalid base chart", "dir", chartPath, "err", err) - return "", "", err, "", "" - } - return chartPath, template, nil, version, pipelineStrategyPath -} func (impl ChartServiceImpl) getRefChartVersion(templateRequest TemplateRequest) (string, error) { var version string if templateRequest.ChartRefId > 0 { - chartRef, err := impl.chartRefRepository.FindById(templateRequest.ChartRefId) + chartRefDto, err := impl.chartRefService.FindById(templateRequest.ChartRefId) if err != nil { - chartRef, err = impl.chartRefRepository.GetDefault() + chartRefDto, err = impl.chartRefService.GetDefault() if err != nil { return "", err } } - version = chartRef.Version + version = chartRefDto.Version } else { - chartRef, err := impl.chartRefRepository.GetDefault() + chartRefDto, err := impl.chartRefService.GetDefault() if err != nil { return "", err } - version = chartRef.Location + version = chartRefDto.Location } return version, nil } @@ -863,31 +754,6 @@ func (impl ChartServiceImpl) UpdateAppOverride(ctx context.Context, templateRequ return templateRequest, nil } -func (impl ChartServiceImpl) handleChartTypeChange(currentLatestChart *chartRepoRepository.Chart, templateRequest *TemplateRequest) (json.RawMessage, error) { - var oldChartRef, newChartRef *chartRepoRepository.ChartRef - var err error - if oldChartRef, err = impl.chartRefRepository.FindById(currentLatestChart.ChartRefId); err != nil { - return nil, fmt.Errorf("chartRef not found for %v", currentLatestChart.ChartRefId) - } - if newChartRef, err = impl.chartRefRepository.FindById(templateRequest.ChartRefId); err != nil { - return nil, fmt.Errorf("chartRef not found for %v", templateRequest.ChartRefId) - } - if len(oldChartRef.Name) == 0 { - oldChartRef.Name = RolloutChartType - } - if len(newChartRef.Name) == 0 { - oldChartRef.Name = RolloutChartType - } - if !CheckCompatibility(oldChartRef.Name, newChartRef.Name) { - return nil, fmt.Errorf("charts are not compatible") - } - updatedOverride, err := PatchWinterSoldierConfig(templateRequest.ValuesOverride, newChartRef.Name) - if err != nil { - return nil, err - } - return updatedOverride, nil -} - type IsReady struct { Flag bool `json:"flag"` Message string `json:"message"` @@ -924,101 +790,25 @@ func (impl ChartServiceImpl) IsReadyToTrigger(appId int, envId int, pipelineId i return isReady, nil } -type ChartRef struct { - Id int `json:"id"` - Version string `json:"version"` - Name string `json:"name"` - Description string `json:"description"` - UserUploaded bool `json:"userUploaded"` - IsAppMetricsSupported bool `json:"isAppMetricsSupported"` -} +func (impl ChartServiceImpl) ChartRefAutocompleteForAppOrEnv(appId int, envId int) (*bean2.ChartRefAutocompleteResponse, error) { + chartRefResponse := &bean2.ChartRefAutocompleteResponse{} + var chartRefs []bean2.ChartRefAutocompleteDto -type ChartRefMetaData struct { - ChartDescription string `json:"chartDescription"` -} - -type ChartRefResponse struct { - ChartRefs []ChartRef `json:"chartRefs"` - LatestChartRef int `json:"latestChartRef"` - LatestAppChartRef int `json:"latestAppChartRef"` - LatestEnvChartRef int `json:"latestEnvChartRef,omitempty"` - ChartsMetadata map[string]ChartRefMetaData `json:"chartMetadata"` // chartName vs Metadata - CompatibleChartTypes []string `json:"compatibleChartTypes,omitempty"` -} - -type ChartYamlStruct struct { - Name string `yaml:"name"` - Version string `yaml:"version"` - Description string `yaml:"description"` -} - -type ChartDataInfo struct { - ChartLocation string `json:"chartLocation"` - ChartName string `json:"chartName"` - ChartVersion string `json:"chartVersion"` - TemporaryFolder string `json:"temporaryFolder"` - Description string `json:"description"` - Message string `json:"message"` -} - -type ChartDto struct { - Id int `json:"id"` - Name string `json:"name"` - ChartDescription string `json:"chartDescription"` - Version string `json:"version"` - IsUserUploaded bool `json:"isUserUploaded"` -} - -func (impl ChartServiceImpl) ChartRefAutocomplete() ([]ChartRef, error) { - var chartRefs []ChartRef - results, err := impl.chartRefRepository.GetAll() + results, err := impl.chartRefService.GetAll() if err != nil { - impl.logger.Errorw("error in fetching chart config", "err", err) - return chartRefs, err - } - - for _, result := range results { - chartRefs = append(chartRefs, ChartRef{ - Id: result.Id, - Version: result.Version, - Description: result.ChartDescription, - UserUploaded: result.UserUploaded, - IsAppMetricsSupported: result.IsAppMetricsSupported, - }) - } - - return chartRefs, nil -} - -func (impl ChartServiceImpl) ChartRefAutocompleteForAppOrEnv(appId int, envId int) (*ChartRefResponse, error) { - chartRefResponse := &ChartRefResponse{ - ChartsMetadata: make(map[string]ChartRefMetaData), - } - var chartRefs []ChartRef - - results, err := impl.chartRefRepository.GetAll() - if err != nil { - impl.logger.Errorw("error in fetching chart config", "err", err) + impl.logger.Errorw("error in fetching chart ref", "err", err) return chartRefResponse, err } - resultsMetadata, err := impl.chartRefRepository.GetAllChartMetadata() + resultsMetadataMap, err := impl.chartRefService.GetAllChartMetadata() if err != nil { impl.logger.Errorw("error in fetching chart metadata", "err", err) return chartRefResponse, err } - for _, resultMetadata := range resultsMetadata { - chartRefMetadata := ChartRefMetaData{ - ChartDescription: resultMetadata.ChartDescription, - } - chartRefResponse.ChartsMetadata[resultMetadata.ChartName] = chartRefMetadata - } + chartRefResponse.ChartsMetadata = resultsMetadataMap var LatestAppChartRef int for _, result := range results { - if len(result.Name) == 0 { - result.Name = "Rollout Deployment" - } - chartRefs = append(chartRefs, ChartRef{ + chartRefs = append(chartRefs, bean2.ChartRefAutocompleteDto{ Id: result.Id, Version: result.Version, Name: result.Name, @@ -1236,13 +1026,13 @@ func (impl ChartServiceImpl) DeploymentTemplateValidate(ctx context.Context, tem } func (impl ChartServiceImpl) JsonSchemaExtractFromFile(chartRefId int) (map[string]interface{}, string, error) { - err := impl.CheckChartExists(chartRefId) + err := impl.chartRefService.CheckChartExists(chartRefId) if err != nil { impl.logger.Errorw("refChartDir Not Found", "err", err) return nil, "", err } - refChartDir, _, err, version, _ := impl.GetRefChart(TemplateRequest{ChartRefId: chartRefId}) + refChartDir, _, err, version, _ := impl.chartRefService.GetRefChart(chartRefId) if err != nil { impl.logger.Errorw("refChartDir Not Found err, JsonSchemaExtractFromFile", err) return nil, "", err @@ -1273,53 +1063,6 @@ func (impl ChartServiceImpl) JsonSchemaExtractFromFile(chartRefId int) (map[stri } } -func (impl ChartServiceImpl) CheckChartExists(chartRefId int) error { - chartRefValue, err := impl.chartRefRepository.FindById(chartRefId) - if err != nil { - impl.logger.Errorw("error in finding ref chart by id", "err", err) - return err - } - refChartLocation := filepath.Join(string(impl.refChartDir), chartRefValue.Location) - if _, err := os.Stat(refChartLocation); os.IsNotExist(err) { - chartInfo, err := impl.ExtractChartIfMissing(chartRefValue.ChartData, string(impl.refChartDir), chartRefValue.Location) - if chartInfo != nil && chartInfo.TemporaryFolder != "" { - err1 := os.RemoveAll(chartInfo.TemporaryFolder) - if err1 != nil { - impl.logger.Errorw("error in deleting temp dir ", "err", err) - } - } - return err - } - return nil -} - -func (impl ChartServiceImpl) CheckIsAppMetricsSupported(chartRefId int) (bool, error) { - chartRefValue, err := impl.chartRefRepository.FindById(chartRefId) - if err != nil { - impl.logger.Errorw("error in finding ref chart by id", "err", err) - return false, nil - } - return chartRefValue.IsAppMetricsSupported, nil -} - -func (impl *ChartServiceImpl) GetLocationFromChartNameAndVersion(chartName string, chartVersion string) string { - var chartLocation string - chartname := impl.FormatChartName(chartName) - chartversion := strings.ReplaceAll(chartVersion, ".", "-") - if !strings.Contains(chartname, chartversion) { - chartLocation = chartname + "_" + chartversion - } else { - chartLocation = chartname - } - return chartLocation -} - -func (impl *ChartServiceImpl) FormatChartName(chartName string) string { - chartname := strings.ReplaceAll(chartName, ".", "-") - chartname = strings.ReplaceAll(chartname, " ", "_") - return chartname -} - func (impl *ChartServiceImpl) ValidateUploadedFileFormat(fileName string) error { if !strings.HasSuffix(fileName, ".tgz") { return errors.New("unsupported format") @@ -1327,243 +1070,14 @@ func (impl *ChartServiceImpl) ValidateUploadedFileFormat(fileName string) error return nil } -func (impl ChartServiceImpl) ReadChartMetaDataForLocation(chartDir string, fileName string) (*ChartYamlStruct, error) { - chartLocation := filepath.Clean(filepath.Join(chartDir, fileName)) - - chartYamlPath := filepath.Clean(filepath.Join(chartLocation, "Chart.yaml")) - if _, err := os.Stat(chartYamlPath); os.IsNotExist(err) { - return nil, fmt.Errorf("Chart.yaml file not present in the directory") - } - - data, err := ioutil.ReadFile(chartYamlPath) - if err != nil { - impl.logger.Errorw("failed reading data from file", "err", err) - return nil, err - } - //println(data) - var chartYaml ChartYamlStruct - err = yaml.Unmarshal(data, &chartYaml) - if err != nil { - impl.logger.Errorw("Unmarshal error of yaml file", "err", err) - return nil, err - } - if chartYaml.Name == "" || chartYaml.Version == "" { - impl.logger.Errorw("Missing values in yaml file either name or version", "err", err) - return nil, errors.New("Missing values in yaml file either name or version") - } - ver := strings.Split(chartYaml.Version, ".") - if len(ver) == 3 { - for _, verObject := range ver { - if _, err := strconv.ParseInt(verObject, 10, 64); err != nil { - return nil, errors.New("Version should contain integers (Ex: 1.1.0)") - } - } - return &chartYaml, nil - } - return nil, errors.New("Version should be of length 3 integers with dot seperated (Ex: 1.1.0)") -} - -func (impl ChartServiceImpl) ExtractChartIfMissing(chartData []byte, refChartDir string, location string) (*ChartDataInfo, error) { - binaryDataReader := bytes.NewReader(chartData) - dir := impl.chartTemplateService.GetDir() - chartInfo := &ChartDataInfo{ - ChartName: "", - ChartVersion: "", - ChartLocation: "", - TemporaryFolder: "", - Description: "", - Message: "", - } - temporaryChartWorkingDir := filepath.Clean(filepath.Join(refChartDir, dir)) - err := os.MkdirAll(temporaryChartWorkingDir, os.ModePerm) - if err != nil { - impl.logger.Errorw("error in creating directory, CallbackConfigMap", "err", err) - return chartInfo, err - } - chartInfo.TemporaryFolder = temporaryChartWorkingDir - err = util2.ExtractTarGz(binaryDataReader, temporaryChartWorkingDir) - if err != nil { - impl.logger.Errorw("error in extracting binary data of charts", "err", err) - return chartInfo, err - } - - var chartLocation string - var chartName string - var chartVersion string - var fileName string - - files, err := ioutil.ReadDir(temporaryChartWorkingDir) - if err != nil { - impl.logger.Errorw("error in reading err dir", "err", err) - return chartInfo, err - } - - fileName = files[0].Name() - if strings.HasPrefix(files[0].Name(), ".") { - fileName = files[1].Name() - } - - currentChartWorkingDir := filepath.Clean(filepath.Join(temporaryChartWorkingDir, fileName)) - - if location == "" { - chartYaml, err := impl.ReadChartMetaDataForLocation(temporaryChartWorkingDir, fileName) - var errorList error - if err != nil { - impl.logger.Errorw("Chart yaml file or content not found") - errorList = err - } - - err = util2.CheckForMissingFiles(currentChartWorkingDir) - if err != nil { - impl.logger.Errorw("Missing files in the folder", "err", err) - if errorList != nil { - errorList = errors.New(errorList.Error() + "; " + err.Error()) - } else { - errorList = err - } - - } - - if errorList != nil { - return chartInfo, errorList - } - - chartName = chartYaml.Name - chartVersion = chartYaml.Version - chartInfo.Description = chartYaml.Description - chartLocation = impl.GetLocationFromChartNameAndVersion(chartName, chartVersion) - location = chartLocation - - // Validate: chart name shouldn't conflict with Devtron charts (no user uploaded chart names should contain any devtron chart names as the prefix) - isReservedChart, _ := impl.ValidateReservedChartName(chartName) - if isReservedChart { - impl.logger.Errorw("request err, chart name is reserved by Devtron") - err = &util.ApiError{ - Code: constants.ChartNameAlreadyReserved, - InternalMessage: CHART_NAME_RESERVED_INTERNAL_ERROR, - UserMessage: fmt.Sprintf("The name '%s' is reserved for a chart provided by Devtron", chartName), - } - return chartInfo, err - } - - // Validate: chart location should be unique - exists, err := impl.chartRefRepository.CheckIfDataExists(location) - if err != nil { - impl.logger.Errorw("Error in searching the database") - return chartInfo, err - } - if exists { - impl.logger.Errorw("request err, chart name and version exists already in the database") - err = &util.ApiError{ - Code: constants.ChartCreatedAlreadyExists, - InternalMessage: CHART_ALREADY_EXISTS_INTERNAL_ERROR, - UserMessage: fmt.Sprintf("%s of %s exists already in the database", chartVersion, chartName), - } - return chartInfo, err - } - - //User Info Message: uploading new version of the existing chart name - existingChart, err := impl.chartRefRepository.FetchChart(chartName) - if err == nil && existingChart != nil { - chartInfo.Message = "New Version detected for " + existingChart[0].Name - } - - } else { - err = dirCopy.Copy(currentChartWorkingDir, filepath.Clean(filepath.Join(refChartDir, location))) - if err != nil { - impl.logger.Errorw("error in copying chart from temp dir to ref chart dir", "err", err) - return chartInfo, err - } - } - - chartInfo.ChartLocation = location - chartInfo.ChartName = chartName - chartInfo.ChartVersion = chartVersion - return chartInfo, nil -} - -func (impl ChartServiceImpl) ValidateReservedChartName(chartName string) (isReservedChart bool, err error) { - formattedChartName := impl.FormatChartName(chartName) - for _, reservedChart := range *ReservedChartRefNamesList { - isReservedChart = (reservedChart.LocationPrefix != "" && strings.HasPrefix(formattedChartName, reservedChart.LocationPrefix)) || - (reservedChart.Name != "" && strings.HasPrefix(strings.ToLower(chartName), reservedChart.Name)) - if isReservedChart { - return true, nil - } - } - return false, nil -} - -func (impl ChartServiceImpl) FetchCustomChartsInfo() ([]*ChartDto, error) { - resultsMetadata, err := impl.chartRefRepository.GetAllChartMetadata() - if err != nil { - impl.logger.Errorw("error in fetching chart metadata", "err", err) - return nil, err - } - chartsMetadata := make(map[string]string) - for _, resultMetadata := range resultsMetadata { - chartsMetadata[resultMetadata.ChartName] = resultMetadata.ChartDescription - } - repo, err := impl.chartRefRepository.GetAll() - if err != nil { - return nil, err - } - var chartDtos []*ChartDto - for _, ref := range repo { - if len(ref.Name) == 0 { - ref.Name = RolloutChartType - } - if description, ok := chartsMetadata[ref.Name]; ref.ChartDescription == "" && ok { - ref.ChartDescription = description - } - chartDto := &ChartDto{ - Id: ref.Id, - Name: ref.Name, - ChartDescription: ref.ChartDescription, - Version: ref.Version, - IsUserUploaded: ref.UserUploaded, - } - chartDtos = append(chartDtos, chartDto) - } - return chartDtos, err -} - -func (impl ChartServiceImpl) CheckCustomChartByAppId(id int) (bool, error) { +func (impl ChartServiceImpl) CheckIfChartRefUserUploadedByAppId(id int) (bool, error) { chartInfo, err := impl.chartRepository.FindLatestChartForAppByAppId(id) if err != nil { return false, err } - chartData, err := impl.chartRefRepository.FindById(chartInfo.ChartRefId) + chartData, err := impl.chartRefService.FindById(chartInfo.ChartRefId) if err != nil { return false, err } return chartData.UserUploaded, err } - -func (impl ChartServiceImpl) CheckCustomChartByChartId(id int) (bool, error) { - chartData, err := impl.chartRefRepository.FindById(id) - if err != nil { - return false, err - } - return chartData.UserUploaded, nil -} - -func (impl ChartServiceImpl) GetCustomChartInBytes(chartRefId int) ([]byte, error) { - chartRef, err := impl.chartRefRepository.FindById(chartRefId) - if err != nil { - impl.logger.Errorw("error getting chart data", "chartRefId", chartRefId, "err", err) - return nil, err - } - // For user uploaded charts ChartData will be retrieved from DB - if chartRef.ChartData != nil { - return chartRef.ChartData, nil - } - // For Devtron reference charts the chart will be load from the directory location - refChartPath := filepath.Join(string(impl.refChartDir), chartRef.Location) - manifestByteArr, err := impl.chartTemplateService.LoadChartInBytes(refChartPath, false) - if err != nil { - impl.logger.Errorw("error in converting chart to bytes", "err", err) - return nil, err - } - return manifestByteArr, nil -} diff --git a/pkg/chart/ChartUtils.go b/pkg/chart/ChartUtils.go index 0d8b180b01..4702060352 100644 --- a/pkg/chart/ChartUtils.go +++ b/pkg/chart/ChartUtils.go @@ -2,8 +2,7 @@ package chart import ( "encoding/json" - chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" - "strings" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/bean" ) func PatchWinterSoldierConfig(override json.RawMessage, newChartType string) (json.RawMessage, error) { @@ -38,9 +37,9 @@ func PatchWinterSoldierIfExists(newChartType string, jsonMap map[string]json.Raw return jsonMap, nil } switch newChartType { - case DeploymentChartType: + case bean.DeploymentChartType: winterSoldierUnmarshalled["type"] = json.RawMessage("\"Deployment\"") - case RolloutChartType: + case bean.RolloutChartType: winterSoldierUnmarshalled["type"] = json.RawMessage("\"Rollout\"") } @@ -52,20 +51,6 @@ func PatchWinterSoldierIfExists(newChartType string, jsonMap map[string]json.Raw return jsonMap, nil } -func SetReservedChartList(devtronChartList []*chartRepoRepository.ChartRef) { - reservedChartRefNamesList := []ReservedChartList{ - {Name: strings.ToLower(RolloutChartType), LocationPrefix: ""}, - {Name: "", LocationPrefix: ReferenceChart}, - } - for _, devtronChart := range devtronChartList { - reservedChartRefNamesList = append(reservedChartRefNamesList, ReservedChartList{ - Name: strings.ToLower(devtronChart.Name), - LocationPrefix: strings.Split(devtronChart.Location, "_")[0], - }) - } - ReservedChartRefNamesList = &reservedChartRefNamesList -} - //func IsFlaggerCanaryEnabled(override json.RawMessage) (bool, error) { // //} diff --git a/pkg/chart/bean.go b/pkg/chart/bean.go index c3dce0a3b1..fbedc8b0c2 100644 --- a/pkg/chart/bean.go +++ b/pkg/chart/bean.go @@ -5,11 +5,6 @@ import ( "github.com/devtron-labs/devtron/internal/sql/models" ) -const ( - CHART_ALREADY_EXISTS_INTERNAL_ERROR = "Chart exists already, try uploading another chart" - CHART_NAME_RESERVED_INTERNAL_ERROR = "Change the name of the chart and try uploading again" -) - var ReservedChartRefNamesList *[]ReservedChartList type ReservedChartList struct { diff --git a/pkg/chartRepo/repository/ChartRefRepository.go b/pkg/chartRepo/repository/ChartRefRepository.go index 4c4c9adfc1..33f9d69fc5 100644 --- a/pkg/chartRepo/repository/ChartRefRepository.go +++ b/pkg/chartRepo/repository/ChartRefRepository.go @@ -7,7 +7,10 @@ import ( "strings" ) -type RefChartDir string +const ( + RefChartDirPath = "scripts/devtron-reference-helm-charts" +) + type ChartRef struct { tableName struct{} `sql:"chart_ref" pg:",discard_unknown_columns"` Id int `sql:"id,pk"` diff --git a/pkg/deployment/manifest/deployedAppMetrics/DeployedAppMetrics.go b/pkg/deployment/manifest/deployedAppMetrics/DeployedAppMetrics.go index b04de22cff..d62d5a5fd1 100644 --- a/pkg/deployment/manifest/deployedAppMetrics/DeployedAppMetrics.go +++ b/pkg/deployment/manifest/deployedAppMetrics/DeployedAppMetrics.go @@ -4,8 +4,8 @@ import ( "context" interalRepo "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/util" - chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/bean" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" "github.com/devtron-labs/devtron/pkg/sql" "github.com/go-pg/pg" "go.opentelemetry.io/otel" @@ -23,20 +23,20 @@ type DeployedAppMetricsService interface { type DeployedAppMetricsServiceImpl struct { logger *zap.SugaredLogger - chartRefRepository chartRepoRepository.ChartRefRepository appLevelMetricsRepository interalRepo.AppLevelMetricsRepository envLevelMetricsRepository interalRepo.EnvLevelAppMetricsRepository + chartRefService chartRef.ChartRefService } func NewDeployedAppMetricsServiceImpl(logger *zap.SugaredLogger, - chartRefRepository chartRepoRepository.ChartRefRepository, appLevelMetricsRepository interalRepo.AppLevelMetricsRepository, - envLevelMetricsRepository interalRepo.EnvLevelAppMetricsRepository) *DeployedAppMetricsServiceImpl { + envLevelMetricsRepository interalRepo.EnvLevelAppMetricsRepository, + chartRefService chartRef.ChartRefService) *DeployedAppMetricsServiceImpl { return &DeployedAppMetricsServiceImpl{ logger: logger, - chartRefRepository: chartRefRepository, appLevelMetricsRepository: appLevelMetricsRepository, envLevelMetricsRepository: envLevelMetricsRepository, + chartRefService: chartRefService, } } @@ -134,7 +134,7 @@ func (impl *DeployedAppMetricsServiceImpl) DeleteEnvLevelMetricsIfPresent(appId, } func (impl *DeployedAppMetricsServiceImpl) checkIsAppMetricsSupported(chartRefId int) (bool, error) { - chartRefValue, err := impl.chartRefRepository.FindById(chartRefId) + chartRefValue, err := impl.chartRefService.FindById(chartRefId) if err != nil { impl.logger.Errorw("error in finding reference chart by id", "err", err) return false, nil diff --git a/pkg/chart/ChartCompatibility.go b/pkg/deployment/manifest/deploymentTemplate/chartRef/ChartCompatibility.go similarity index 67% rename from pkg/chart/ChartCompatibility.go rename to pkg/deployment/manifest/deploymentTemplate/chartRef/ChartCompatibility.go index 4c1fb3cd71..cee6fcfff9 100644 --- a/pkg/chart/ChartCompatibility.go +++ b/pkg/deployment/manifest/deploymentTemplate/chartRef/ChartCompatibility.go @@ -1,16 +1,12 @@ -package chart +package chartRef -type stringSet map[string]struct{} +import "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/bean" -const ( - DeploymentChartType = "Deployment" - RolloutChartType = "Rollout Deployment" - ReferenceChart = "reference-chart" -) +type stringSet map[string]struct{} var chartCompatibilityMatrix = map[string]stringSet{ - DeploymentChartType: {RolloutChartType: {}, DeploymentChartType: {}}, - RolloutChartType: {DeploymentChartType: {}, RolloutChartType: {}}, + bean.DeploymentChartType: {bean.RolloutChartType: {}, bean.DeploymentChartType: {}}, + bean.RolloutChartType: {bean.DeploymentChartType: {}, bean.RolloutChartType: {}}, } func CheckCompatibility(oldChartType, newChartType string) bool { diff --git a/pkg/deployment/manifest/deploymentTemplate/chartRef/ChartRefService.go b/pkg/deployment/manifest/deploymentTemplate/chartRef/ChartRefService.go new file mode 100644 index 0000000000..0bfd0bd278 --- /dev/null +++ b/pkg/deployment/manifest/deploymentTemplate/chartRef/ChartRefService.go @@ -0,0 +1,494 @@ +package chartRef + +import ( + "bytes" + "errors" + "fmt" + "github.com/devtron-labs/devtron/internal/constants" + "github.com/devtron-labs/devtron/internal/util" + chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/adapter" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/bean" + util2 "github.com/devtron-labs/devtron/util" + dirCopy "github.com/otiai10/copy" + "go.uber.org/zap" + "io/ioutil" + "k8s.io/helm/pkg/chartutil" + "os" + "path" + "path/filepath" + "sigs.k8s.io/yaml" + "strconv" + "strings" +) + +type ChartRefService interface { + GetDefault() (*bean.ChartRefDto, error) + GetAll() ([]*bean.ChartRefDto, error) + GetAllChartMetadata() (map[string]bean.ChartRefMetaData, error) + FindById(chartRefId int) (*bean.ChartRefDto, error) + FindByVersionAndName(version, name string) (*bean.ChartRefDto, error) + FetchInfoOfChartConfiguredInApp(appId int) (*bean.ChartRefDto, error) + ChartRefAutocomplete() ([]*bean.ChartRefAutocompleteDto, error) + + SaveCustomChart(req *bean.CustomChartRefDto) error + FetchCustomChartsInfo() ([]*bean.ChartDto, error) + + ChartRefIdsCompatible(oldChartRefId int, newChartRefId int) (bool, string, string) + + CheckChartExists(chartRefId int) error + GetRefChart(chartRefId int) (string, string, error, string, string) + ExtractChartIfMissing(chartData []byte, refChartDir string, location string) (*bean.ChartDataInfo, error) + GetCustomChartInBytes(chartRefId int) ([]byte, error) +} + +type ChartRefServiceImpl struct { + logger *zap.SugaredLogger + chartRefRepository chartRepoRepository.ChartRefRepository + chartTemplateService util.ChartTemplateService +} + +func NewChartRefServiceImpl(logger *zap.SugaredLogger, + chartRefRepository chartRepoRepository.ChartRefRepository, + chartTemplateService util.ChartTemplateService) *ChartRefServiceImpl { + // cache devtron reference charts list + devtronChartList, _ := chartRefRepository.FetchAllChartInfoByUploadFlag(false) + setReservedChartList(devtronChartList) + return &ChartRefServiceImpl{ + logger: logger, + chartRefRepository: chartRefRepository, + chartTemplateService: chartTemplateService, + } +} + +func (impl *ChartRefServiceImpl) GetDefault() (*bean.ChartRefDto, error) { + chartRef, err := impl.chartRefRepository.GetDefault() + if err != nil { + impl.logger.Errorw("error in getting default chartRef", "err", err) + return nil, err + } + return adapter.ConvertChartRefDbObjToBean(chartRef), nil +} + +func (impl *ChartRefServiceImpl) GetAll() ([]*bean.ChartRefDto, error) { + chartRefs, err := impl.chartRefRepository.GetAll() + if err != nil { + impl.logger.Errorw("error in getting all chartRefs", "err", err) + return nil, err + } + chartRefDtos := make([]*bean.ChartRefDto, 0, len(chartRefs)) + for _, chartRef := range chartRefs { + chartRefDtos = append(chartRefDtos, adapter.ConvertChartRefDbObjToBean(chartRef)) + } + return chartRefDtos, nil +} + +func (impl *ChartRefServiceImpl) GetAllChartMetadata() (map[string]bean.ChartRefMetaData, error) { + chartRefMetadatas, err := impl.chartRefRepository.GetAllChartMetadata() + if err != nil { + impl.logger.Errorw("error in getting all chart metadatas", "err", err) + return nil, err + } + chartsMetadataMap := make(map[string]bean.ChartRefMetaData, len(chartRefMetadatas)) + for _, chartRefMetadata := range chartRefMetadatas { + metadataDto := bean.ChartRefMetaData{ + ChartDescription: chartRefMetadata.ChartDescription, + } + chartsMetadataMap[chartRefMetadata.ChartName] = metadataDto + } + return chartsMetadataMap, nil +} +func (impl *ChartRefServiceImpl) ChartRefIdsCompatible(oldChartRefId int, newChartRefId int) (bool, string, string) { + oldChart, err := impl.FindById(oldChartRefId) + if err != nil { + return false, "", "" + } + newChart, err := impl.FindById(newChartRefId) + if err != nil { + return false, "", "" + } + return CheckCompatibility(oldChart.Name, newChart.Name), oldChart.Name, newChart.Name +} + +func (impl *ChartRefServiceImpl) FindById(chartRefId int) (*bean.ChartRefDto, error) { + chartRef, err := impl.chartRefRepository.FindById(chartRefId) + if err != nil { + impl.logger.Errorw("error in getting chartRef by id", "err", err, "chartRefId", chartRefId) + return nil, err + } + return adapter.ConvertChartRefDbObjToBean(chartRef), nil +} + +func (impl *ChartRefServiceImpl) FindByVersionAndName(version, name string) (*bean.ChartRefDto, error) { + chartRef, err := impl.chartRefRepository.FindByVersionAndName(name, version) + if err != nil { + impl.logger.Errorw("error in getting chartRef by version and name", "err", err, "version", version, "name", name) + return nil, err + } + return adapter.ConvertChartRefDbObjToBean(chartRef), nil +} + +func (impl *ChartRefServiceImpl) FetchInfoOfChartConfiguredInApp(appId int) (*bean.ChartRefDto, error) { + chartRef, err := impl.chartRefRepository.FetchInfoOfChartConfiguredInApp(appId) + if err != nil { + impl.logger.Errorw("error in getting chart info for chart configured in app", "err", err, "appId", appId) + return nil, err + } + return adapter.ConvertChartRefDbObjToBean(chartRef), nil +} + +func (impl *ChartRefServiceImpl) SaveCustomChart(req *bean.CustomChartRefDto) error { + chartRefDbObj := adapter.ConvertCustomChartRefDtoToDbObj(req) + err := impl.chartRefRepository.Save(chartRefDbObj) + if err != nil { + impl.logger.Errorw("error in saving chart ref", "err", err, "chartRef", chartRefDbObj) + return err + } + return nil +} + +func (impl *ChartRefServiceImpl) GetRefChart(chartRefId int) (string, string, error, string, string) { + var template string + var version string + //path of file in chart from where strategy config is to be taken + var pipelineStrategyPath string + if chartRefId > 0 { + chartRef, err := impl.chartRefRepository.FindById(chartRefId) + if err != nil { + chartRef, err = impl.chartRefRepository.GetDefault() + if err != nil { + return "", "", err, "", "" + } + } else if chartRef.UserUploaded { + refChartLocation := filepath.Join(chartRepoRepository.RefChartDirPath, chartRef.Location) + if _, err := os.Stat(refChartLocation); os.IsNotExist(err) { + chartInfo, err := impl.ExtractChartIfMissing(chartRef.ChartData, chartRepoRepository.RefChartDirPath, chartRef.Location) + if chartInfo != nil && chartInfo.TemporaryFolder != "" { + err1 := os.RemoveAll(chartInfo.TemporaryFolder) + if err1 != nil { + impl.logger.Errorw("error in deleting temp dir ", "err", err) + } + } + if err != nil { + impl.logger.Errorw("Error regarding uploaded chart", "err", err) + return "", "", err, "", "" + } + + } + } + template = chartRef.Location + version = chartRef.Version + pipelineStrategyPath = chartRef.DeploymentStrategyPath + } else { + chartRef, err := impl.chartRefRepository.GetDefault() + if err != nil { + return "", "", err, "", "" + } + template = chartRef.Location + version = chartRef.Version + pipelineStrategyPath = chartRef.DeploymentStrategyPath + } + + //TODO VIKI- fetch from chart ref table + chartPath := path.Join(chartRepoRepository.RefChartDirPath, template) + valid, err := chartutil.IsChartDir(chartPath) + if err != nil || !valid { + impl.logger.Errorw("invalid base chart", "dir", chartPath, "err", err) + return "", "", err, "", "" + } + return chartPath, template, nil, version, pipelineStrategyPath +} + +func (impl *ChartRefServiceImpl) ChartRefAutocomplete() ([]*bean.ChartRefAutocompleteDto, error) { + var chartRefs []*bean.ChartRefAutocompleteDto + results, err := impl.chartRefRepository.GetAll() + if err != nil { + impl.logger.Errorw("error in fetching chart config", "err", err) + return chartRefs, err + } + + for _, result := range results { + chartRefs = append(chartRefs, &bean.ChartRefAutocompleteDto{ + Id: result.Id, + Version: result.Version, + Description: result.ChartDescription, + UserUploaded: result.UserUploaded, + IsAppMetricsSupported: result.IsAppMetricsSupported, + }) + } + + return chartRefs, nil +} + +func (impl *ChartRefServiceImpl) GetCustomChartInBytes(chartRefId int) ([]byte, error) { + chartRef, err := impl.chartRefRepository.FindById(chartRefId) + if err != nil { + impl.logger.Errorw("error getting chart data", "chartRefId", chartRefId, "err", err) + return nil, err + } + // For user uploaded charts ChartData will be retrieved from DB + if chartRef.ChartData != nil { + return chartRef.ChartData, nil + } + // For Devtron reference charts the chart will be load from the directory location + refChartPath := filepath.Join(chartRepoRepository.RefChartDirPath, chartRef.Location) + manifestByteArr, err := impl.chartTemplateService.LoadChartInBytes(refChartPath, false) + if err != nil { + impl.logger.Errorw("error in converting chart to bytes", "err", err) + return nil, err + } + return manifestByteArr, nil +} + +func (impl *ChartRefServiceImpl) FetchCustomChartsInfo() ([]*bean.ChartDto, error) { + resultsMetadata, err := impl.chartRefRepository.GetAllChartMetadata() + if err != nil { + impl.logger.Errorw("error in fetching chart metadata", "err", err) + return nil, err + } + chartsMetadata := make(map[string]string) + for _, resultMetadata := range resultsMetadata { + chartsMetadata[resultMetadata.ChartName] = resultMetadata.ChartDescription + } + repo, err := impl.chartRefRepository.GetAll() + if err != nil { + return nil, err + } + var chartDtos []*bean.ChartDto + for _, ref := range repo { + if len(ref.Name) == 0 { + ref.Name = bean.RolloutChartType + } + if description, ok := chartsMetadata[ref.Name]; ref.ChartDescription == "" && ok { + ref.ChartDescription = description + } + chartDto := &bean.ChartDto{ + Id: ref.Id, + Name: ref.Name, + ChartDescription: ref.ChartDescription, + Version: ref.Version, + IsUserUploaded: ref.UserUploaded, + } + chartDtos = append(chartDtos, chartDto) + } + return chartDtos, err +} + +func (impl *ChartRefServiceImpl) CheckChartExists(chartRefId int) error { + chartRefValue, err := impl.chartRefRepository.FindById(chartRefId) + if err != nil { + impl.logger.Errorw("error in finding ref chart by id", "err", err) + return err + } + refChartLocation := filepath.Join(chartRepoRepository.RefChartDirPath, chartRefValue.Location) + if _, err := os.Stat(refChartLocation); os.IsNotExist(err) { + chartInfo, err := impl.ExtractChartIfMissing(chartRefValue.ChartData, chartRepoRepository.RefChartDirPath, chartRefValue.Location) + if chartInfo != nil && chartInfo.TemporaryFolder != "" { + err1 := os.RemoveAll(chartInfo.TemporaryFolder) + if err1 != nil { + impl.logger.Errorw("error in deleting temp dir ", "err", err) + } + } + return err + } + return nil +} + +func (impl *ChartRefServiceImpl) ExtractChartIfMissing(chartData []byte, refChartDir string, location string) (*bean.ChartDataInfo, error) { + binaryDataReader := bytes.NewReader(chartData) + dir := impl.chartTemplateService.GetDir() + chartInfo := &bean.ChartDataInfo{ + ChartName: "", + ChartVersion: "", + ChartLocation: "", + TemporaryFolder: "", + Description: "", + Message: "", + } + temporaryChartWorkingDir := filepath.Clean(filepath.Join(refChartDir, dir)) + err := os.MkdirAll(temporaryChartWorkingDir, os.ModePerm) + if err != nil { + impl.logger.Errorw("error in creating directory, CallbackConfigMap", "err", err) + return chartInfo, err + } + chartInfo.TemporaryFolder = temporaryChartWorkingDir + err = util2.ExtractTarGz(binaryDataReader, temporaryChartWorkingDir) + if err != nil { + impl.logger.Errorw("error in extracting binary data of charts", "err", err) + return chartInfo, err + } + + var chartLocation string + var chartName string + var chartVersion string + var fileName string + + files, err := ioutil.ReadDir(temporaryChartWorkingDir) + if err != nil { + impl.logger.Errorw("error in reading err dir", "err", err) + return chartInfo, err + } + + fileName = files[0].Name() + if strings.HasPrefix(files[0].Name(), ".") { + fileName = files[1].Name() + } + + currentChartWorkingDir := filepath.Clean(filepath.Join(temporaryChartWorkingDir, fileName)) + + if location == "" { + chartYaml, err := impl.readChartMetaDataForLocation(temporaryChartWorkingDir, fileName) + var errorList error + if err != nil { + impl.logger.Errorw("Chart yaml file or content not found") + errorList = err + } + + err = util2.CheckForMissingFiles(currentChartWorkingDir) + if err != nil { + impl.logger.Errorw("Missing files in the folder", "err", err) + if errorList != nil { + errorList = errors.New(errorList.Error() + "; " + err.Error()) + } else { + errorList = err + } + + } + + if errorList != nil { + return chartInfo, errorList + } + + chartName = chartYaml.Name + chartVersion = chartYaml.Version + chartInfo.Description = chartYaml.Description + chartLocation = impl.getLocationFromChartNameAndVersion(chartName, chartVersion) + location = chartLocation + + // Validate: chart name shouldn't conflict with Devtron charts (no user uploaded chart names should contain any devtron chart names as the prefix) + isReservedChart, _ := impl.validateReservedChartName(chartName) + if isReservedChart { + impl.logger.Errorw("request err, chart name is reserved by Devtron") + err = &util.ApiError{ + Code: constants.ChartNameAlreadyReserved, + InternalMessage: bean.CHART_NAME_RESERVED_INTERNAL_ERROR, + UserMessage: fmt.Sprintf("The name '%s' is reserved for a chart provided by Devtron", chartName), + } + return chartInfo, err + } + + // Validate: chart location should be unique + exists, err := impl.chartRefRepository.CheckIfDataExists(location) + if err != nil { + impl.logger.Errorw("Error in searching the database") + return chartInfo, err + } + if exists { + impl.logger.Errorw("request err, chart name and version exists already in the database") + err = &util.ApiError{ + Code: constants.ChartCreatedAlreadyExists, + InternalMessage: bean.CHART_ALREADY_EXISTS_INTERNAL_ERROR, + UserMessage: fmt.Sprintf("%s of %s exists already in the database", chartVersion, chartName), + } + return chartInfo, err + } + + //User Info Message: uploading new version of the existing chart name + existingChart, err := impl.chartRefRepository.FetchChart(chartName) + if err == nil && existingChart != nil { + chartInfo.Message = "New Version detected for " + existingChart[0].Name + } + + } else { + err = dirCopy.Copy(currentChartWorkingDir, filepath.Clean(filepath.Join(refChartDir, location))) + if err != nil { + impl.logger.Errorw("error in copying chart from temp dir to ref chart dir", "err", err) + return chartInfo, err + } + } + + chartInfo.ChartLocation = location + chartInfo.ChartName = chartName + chartInfo.ChartVersion = chartVersion + return chartInfo, nil +} + +func (impl *ChartRefServiceImpl) readChartMetaDataForLocation(chartDir string, fileName string) (*bean.ChartYamlStruct, error) { + chartLocation := filepath.Clean(filepath.Join(chartDir, fileName)) + + chartYamlPath := filepath.Clean(filepath.Join(chartLocation, "Chart.yaml")) + if _, err := os.Stat(chartYamlPath); os.IsNotExist(err) { + return nil, fmt.Errorf("Chart.yaml file not present in the directory") + } + + data, err := ioutil.ReadFile(chartYamlPath) + if err != nil { + impl.logger.Errorw("failed reading data from file", "err", err) + return nil, err + } + //println(data) + var chartYaml bean.ChartYamlStruct + err = yaml.Unmarshal(data, &chartYaml) + if err != nil { + impl.logger.Errorw("Unmarshal error of yaml file", "err", err) + return nil, err + } + if chartYaml.Name == "" || chartYaml.Version == "" { + impl.logger.Errorw("Missing values in yaml file either name or version", "err", err) + return nil, errors.New("Missing values in yaml file either name or version") + } + ver := strings.Split(chartYaml.Version, ".") + if len(ver) == 3 { + for _, verObject := range ver { + if _, err := strconv.ParseInt(verObject, 10, 64); err != nil { + return nil, errors.New("Version should contain integers (Ex: 1.1.0)") + } + } + return &chartYaml, nil + } + return nil, errors.New("Version should be of length 3 integers with dot seperated (Ex: 1.1.0)") +} + +func (impl *ChartRefServiceImpl) validateReservedChartName(chartName string) (isReservedChart bool, err error) { + formattedChartName := impl.formatChartName(chartName) + for _, reservedChart := range *bean.ReservedChartRefNamesList { + isReservedChart = (reservedChart.LocationPrefix != "" && strings.HasPrefix(formattedChartName, reservedChart.LocationPrefix)) || + (reservedChart.Name != "" && strings.HasPrefix(strings.ToLower(chartName), reservedChart.Name)) + if isReservedChart { + return true, nil + } + } + return false, nil +} + +func (impl *ChartRefServiceImpl) getLocationFromChartNameAndVersion(chartName string, chartVersion string) string { + var chartLocation string + chartname := impl.formatChartName(chartName) + chartversion := strings.ReplaceAll(chartVersion, ".", "-") + if !strings.Contains(chartname, chartversion) { + chartLocation = chartname + "_" + chartversion + } else { + chartLocation = chartname + } + return chartLocation +} + +func (impl *ChartRefServiceImpl) formatChartName(chartName string) string { + chartname := strings.ReplaceAll(chartName, ".", "-") + chartname = strings.ReplaceAll(chartname, " ", "_") + return chartname +} + +func setReservedChartList(devtronChartList []*chartRepoRepository.ChartRef) { + reservedChartRefNamesList := []bean.ReservedChartList{ + {Name: strings.ToLower(bean.RolloutChartType), LocationPrefix: ""}, + {Name: "", LocationPrefix: bean.ReferenceChart}, + } + for _, devtronChart := range devtronChartList { + reservedChartRefNamesList = append(reservedChartRefNamesList, bean.ReservedChartList{ + Name: strings.ToLower(devtronChart.Name), + LocationPrefix: strings.Split(devtronChart.Location, "_")[0], + }) + } + bean.ReservedChartRefNamesList = &reservedChartRefNamesList +} diff --git a/pkg/deployment/manifest/deploymentTemplate/chartRef/adapter/adapter.go b/pkg/deployment/manifest/deploymentTemplate/chartRef/adapter/adapter.go new file mode 100644 index 0000000000..6e2ec31658 --- /dev/null +++ b/pkg/deployment/manifest/deploymentTemplate/chartRef/adapter/adapter.go @@ -0,0 +1,45 @@ +package adapter + +import ( + chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/bean" +) + +func ConvertChartRefDbObjToBean(ref *chartRepoRepository.ChartRef) *bean.ChartRefDto { + dto := &bean.ChartRefDto{ + Id: ref.Id, + Location: ref.Location, + Version: ref.Version, + Default: ref.Default, + ChartData: ref.ChartData, + ChartDescription: ref.ChartDescription, + UserUploaded: ref.UserUploaded, + IsAppMetricsSupported: ref.IsAppMetricsSupported, + DeploymentStrategyPath: ref.DeploymentStrategyPath, + JsonPathForStrategy: ref.JsonPathForStrategy, + } + if len(ref.Name) == 0 { + dto.Name = bean.RolloutChartType + } else { + dto.Name = ref.Name + } + return dto +} + +func ConvertCustomChartRefDtoToDbObj(ref *bean.CustomChartRefDto) *chartRepoRepository.ChartRef { + return &chartRepoRepository.ChartRef{ + Id: ref.Id, + Location: ref.Location, + Version: ref.Version, + Active: ref.Active, + Default: ref.Default, + Name: ref.Name, + ChartData: ref.ChartData, + ChartDescription: ref.ChartDescription, + UserUploaded: ref.UserUploaded, + IsAppMetricsSupported: ref.IsAppMetricsSupported, + DeploymentStrategyPath: ref.DeploymentStrategyPath, + JsonPathForStrategy: ref.JsonPathForStrategy, + AuditLog: ref.AuditLog, + } +} diff --git a/pkg/deployment/manifest/deploymentTemplate/chartRef/bean/bean.go b/pkg/deployment/manifest/deploymentTemplate/chartRef/bean/bean.go new file mode 100644 index 0000000000..b0f662e04e --- /dev/null +++ b/pkg/deployment/manifest/deploymentTemplate/chartRef/bean/bean.go @@ -0,0 +1,95 @@ +package bean + +import "github.com/devtron-labs/devtron/pkg/sql" + +const ( + DeploymentChartType = "Deployment" + RolloutChartType = "Rollout Deployment" + ReferenceChart = "reference-chart" + CHART_ALREADY_EXISTS_INTERNAL_ERROR = "Chart exists already, try uploading another chart" + CHART_NAME_RESERVED_INTERNAL_ERROR = "Change the name of the chart and try uploading again" +) + +type ChartDataInfo struct { + ChartLocation string `json:"chartLocation"` + ChartName string `json:"chartName"` + ChartVersion string `json:"chartVersion"` + TemporaryFolder string `json:"temporaryFolder"` + Description string `json:"description"` + Message string `json:"message"` +} + +type ChartYamlStruct struct { + Name string `yaml:"name"` + Version string `yaml:"version"` + Description string `yaml:"description"` +} + +var ReservedChartRefNamesList *[]ReservedChartList + +type ReservedChartList struct { + LocationPrefix string + Name string +} + +type ChartRefDto struct { + Id int `json:"id"` + Location string `json:"location"` + Version string `json:"version"` + Default bool `json:"isDefault"` + Name string `json:"name"` + ChartData []byte `json:"chartData"` + ChartDescription string `json:"chartDescription"` + UserUploaded bool `json:"userUploaded,notnull"` + IsAppMetricsSupported bool `json:"isAppMetricsSupported"` + DeploymentStrategyPath string `json:"deploymentStrategyPath"` + JsonPathForStrategy string `json:"jsonPathForStrategy"` +} + +// TODO: below objects are created/moved while refactoring to remove db object usage, to remove/replace them with the common objects mentioned above + +type CustomChartRefDto struct { + Id int `sql:"id,pk"` + Location string `sql:"location"` + Version string `sql:"version"` + Active bool `sql:"active,notnull"` + Default bool `sql:"is_default,notnull"` + Name string `sql:"name"` + ChartData []byte `sql:"chart_data"` + ChartDescription string `sql:"chart_description"` + UserUploaded bool `sql:"user_uploaded,notnull"` + IsAppMetricsSupported bool `sql:"is_app_metrics_supported,notnull"` + DeploymentStrategyPath string `sql:"deployment_strategy_path"` + JsonPathForStrategy string `sql:"json_path_for_strategy"` + sql.AuditLog +} + +type ChartRefAutocompleteDto struct { + Id int `json:"id"` + Version string `json:"version"` + Name string `json:"name"` + Description string `json:"description"` + UserUploaded bool `json:"userUploaded"` + IsAppMetricsSupported bool `json:"isAppMetricsSupported"` +} + +type ChartRefMetaData struct { + ChartDescription string `json:"chartDescription"` +} + +type ChartRefAutocompleteResponse struct { + ChartRefs []ChartRefAutocompleteDto `json:"chartRefs"` + LatestChartRef int `json:"latestChartRef"` + LatestAppChartRef int `json:"latestAppChartRef"` + LatestEnvChartRef int `json:"latestEnvChartRef,omitempty"` + ChartsMetadata map[string]ChartRefMetaData `json:"chartMetadata"` // chartName vs Metadata + CompatibleChartTypes []string `json:"compatibleChartTypes,omitempty"` +} + +type ChartDto struct { + Id int `json:"id"` + Name string `json:"name"` + ChartDescription string `json:"chartDescription"` + Version string `json:"version"` + IsUserUploaded bool `json:"isUserUploaded"` +} diff --git a/pkg/deployment/manifest/wire_deployment_manifest.go b/pkg/deployment/manifest/wire_deployment_manifest.go new file mode 100644 index 0000000000..fe6d1a6e61 --- /dev/null +++ b/pkg/deployment/manifest/wire_deployment_manifest.go @@ -0,0 +1,17 @@ +package manifest + +import ( + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" + "github.com/google/wire" +) + +var DeploymentManifestWireSet = wire.NewSet( + deployedAppMetrics.NewDeployedAppMetricsServiceImpl, + wire.Bind(new(deployedAppMetrics.DeployedAppMetricsService), new(*deployedAppMetrics.DeployedAppMetricsServiceImpl)), + deploymentTemplate.NewDeploymentTemplateServiceImpl, + wire.Bind(new(deploymentTemplate.DeploymentTemplateService), new(*deploymentTemplate.DeploymentTemplateServiceImpl)), + chartRef.NewChartRefServiceImpl, + wire.Bind(new(chartRef.ChartRefService), new(*chartRef.ChartRefServiceImpl)), +) diff --git a/pkg/generateManifest/DeployementTemplateService.go b/pkg/generateManifest/DeployementTemplateService.go index be120b5daa..1cb3023bf7 100644 --- a/pkg/generateManifest/DeployementTemplateService.go +++ b/pkg/generateManifest/DeployementTemplateService.go @@ -13,6 +13,7 @@ import ( "github.com/devtron-labs/devtron/pkg/chart" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" repository3 "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/history" "github.com/devtron-labs/devtron/pkg/resourceQualifiers" @@ -84,6 +85,7 @@ type DeploymentTemplateServiceImpl struct { environmentRepository repository3.EnvironmentRepository appRepository appRepository.AppRepository scopedVariableManager variables.ScopedVariableManager + chartRefService chartRef.ChartRefService } func NewDeploymentTemplateServiceImpl(Logger *zap.SugaredLogger, chartService chart.ChartService, @@ -100,7 +102,7 @@ func NewDeploymentTemplateServiceImpl(Logger *zap.SugaredLogger, chartService ch environmentRepository repository3.EnvironmentRepository, appRepository appRepository.AppRepository, scopedVariableManager variables.ScopedVariableManager, -) *DeploymentTemplateServiceImpl { + chartRefService chartRef.ChartRefService) *DeploymentTemplateServiceImpl { return &DeploymentTemplateServiceImpl{ Logger: Logger, chartService: chartService, @@ -117,6 +119,7 @@ func NewDeploymentTemplateServiceImpl(Logger *zap.SugaredLogger, chartService ch environmentRepository: environmentRepository, appRepository: appRepository, scopedVariableManager: scopedVariableManager, + chartRefService: chartRefService, } } @@ -301,7 +304,7 @@ func (impl DeploymentTemplateServiceImpl) extractScopeData(request DeploymentTem } func (impl DeploymentTemplateServiceImpl) GenerateManifest(ctx context.Context, chartRefId int, valuesYaml string) (*openapi2.TemplateChartResponse, error) { - refChart, template, err, version, _ := impl.chartService.GetRefChart(chart.TemplateRequest{ChartRefId: chartRefId}) + refChart, template, err, version, _ := impl.chartRefService.GetRefChart(chartRefId) if err != nil { impl.Logger.Errorw("error in getting refChart", "err", err, "chartRefId", chartRefId) return nil, err diff --git a/pkg/pipeline/DeploymentConfigService.go b/pkg/pipeline/DeploymentConfigService.go index 1e5a57c615..f12c90c9e0 100644 --- a/pkg/pipeline/DeploymentConfigService.go +++ b/pkg/pipeline/DeploymentConfigService.go @@ -8,6 +8,7 @@ import ( "github.com/devtron-labs/devtron/pkg/bean" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" "github.com/devtron-labs/devtron/pkg/pipeline/history" repository2 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" "github.com/devtron-labs/devtron/pkg/resourceQualifiers" @@ -31,9 +32,9 @@ type DeploymentConfigServiceImpl struct { pipelineConfigRepository chartConfig.PipelineConfigRepository configMapRepository chartConfig.ConfigMapRepository configMapHistoryService history.ConfigMapHistoryService - chartRefRepository chartRepoRepository.ChartRefRepository scopedVariableManager variables.ScopedVariableCMCSManager deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService + chartRefService chartRef.ChartRefService } func NewDeploymentConfigServiceImpl(logger *zap.SugaredLogger, @@ -43,9 +44,9 @@ func NewDeploymentConfigServiceImpl(logger *zap.SugaredLogger, pipelineConfigRepository chartConfig.PipelineConfigRepository, configMapRepository chartConfig.ConfigMapRepository, configMapHistoryService history.ConfigMapHistoryService, - chartRefRepository chartRepoRepository.ChartRefRepository, scopedVariableManager variables.ScopedVariableCMCSManager, - deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService) *DeploymentConfigServiceImpl { + deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService, + chartRefService chartRef.ChartRefService) *DeploymentConfigServiceImpl { return &DeploymentConfigServiceImpl{ logger: logger, envConfigOverrideRepository: envConfigOverrideRepository, @@ -54,9 +55,9 @@ func NewDeploymentConfigServiceImpl(logger *zap.SugaredLogger, pipelineConfigRepository: pipelineConfigRepository, configMapRepository: configMapRepository, configMapHistoryService: configMapHistoryService, - chartRefRepository: chartRefRepository, scopedVariableManager: scopedVariableManager, deployedAppMetricsService: deployedAppMetricsService, + chartRefService: chartRefService, } } @@ -107,7 +108,7 @@ func (impl *DeploymentConfigServiceImpl) GetLatestDeploymentTemplateConfig(ctx c deploymentTemplateConfig := &history.HistoryDetailDto{} if envOverride != nil && envOverride.Id > 0 && envOverride.IsOverride { if envOverride.Chart != nil { - chartRef, err := impl.chartRefRepository.FindById(envOverride.Chart.ChartRefId) + chartRefDto, err := impl.chartRefService.FindById(envOverride.Chart.ChartRefId) if err != nil { impl.logger.Errorw("error in getting chartRef by id", "err", err, "chartRefId", envOverride.Chart.ChartRefId) return nil, err @@ -132,7 +133,7 @@ func (impl *DeploymentConfigServiceImpl) GetLatestDeploymentTemplateConfig(ctx c deploymentTemplateConfig = &history.HistoryDetailDto{ TemplateName: envOverride.Chart.ChartName, - TemplateVersion: chartRef.Version, + TemplateVersion: chartRefDto.Version, IsAppMetricsEnabled: &isAppMetricsEnabled, CodeEditorValue: &history.HistoryDetailConfig{ DisplayName: "values.yaml", @@ -148,7 +149,7 @@ func (impl *DeploymentConfigServiceImpl) GetLatestDeploymentTemplateConfig(ctx c impl.logger.Errorw("error in getting chart by appId", "err", err, "appId", pipeline.AppId) return nil, err } - chartRef, err := impl.chartRefRepository.FindById(chart.ChartRefId) + chartRefDto, err := impl.chartRefService.FindById(chart.ChartRefId) if err != nil { impl.logger.Errorw("error in getting chartRef by id", "err", err, "chartRefId", envOverride.Chart.ChartRefId) return nil, err @@ -173,7 +174,7 @@ func (impl *DeploymentConfigServiceImpl) GetLatestDeploymentTemplateConfig(ctx c } deploymentTemplateConfig = &history.HistoryDetailDto{ TemplateName: chart.ChartName, - TemplateVersion: chartRef.Version, + TemplateVersion: chartRefDto.Version, IsAppMetricsEnabled: &isAppMetricsEnabled, CodeEditorValue: &history.HistoryDetailConfig{ DisplayName: "values.yaml", diff --git a/pkg/pipeline/PropertiesConfig.go b/pkg/pipeline/PropertiesConfig.go index 686042bd65..24bdf83fc4 100644 --- a/pkg/pipeline/PropertiesConfig.go +++ b/pkg/pipeline/PropertiesConfig.go @@ -34,10 +34,8 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline/history" "github.com/devtron-labs/devtron/pkg/sql" - "github.com/devtron-labs/devtron/client/argocdServer/application" "github.com/devtron-labs/devtron/internal/sql/models" "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" - "github.com/devtron-labs/devtron/internal/util" "github.com/go-pg/pg" "github.com/juju/errors" "go.uber.org/zap" @@ -62,11 +60,7 @@ type PropertiesConfigServiceImpl struct { logger *zap.SugaredLogger envConfigRepo chartConfig.EnvConfigOverrideRepository chartRepo chartRepoRepository.ChartRepository - chartRefRepository chartRepoRepository.ChartRefRepository - mergeUtil util.MergeUtil environmentRepository repository2.EnvironmentRepository - ciCdPipelineOrchestrator CiCdPipelineOrchestrator - application application.ServiceClient deploymentTemplateHistoryService history.DeploymentTemplateHistoryService scopedVariableManager variables.ScopedVariableManager deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService @@ -75,11 +69,7 @@ type PropertiesConfigServiceImpl struct { func NewPropertiesConfigServiceImpl(logger *zap.SugaredLogger, envConfigRepo chartConfig.EnvConfigOverrideRepository, chartRepo chartRepoRepository.ChartRepository, - chartRefRepository chartRepoRepository.ChartRefRepository, - mergeUtil util.MergeUtil, environmentRepository repository2.EnvironmentRepository, - ciCdPipelineOrchestrator CiCdPipelineOrchestrator, - application application.ServiceClient, deploymentTemplateHistoryService history.DeploymentTemplateHistoryService, scopedVariableManager variables.ScopedVariableManager, deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService) *PropertiesConfigServiceImpl { @@ -87,11 +77,7 @@ func NewPropertiesConfigServiceImpl(logger *zap.SugaredLogger, logger: logger, envConfigRepo: envConfigRepo, chartRepo: chartRepo, - chartRefRepository: chartRefRepository, - mergeUtil: mergeUtil, environmentRepository: environmentRepository, - ciCdPipelineOrchestrator: ciCdPipelineOrchestrator, - application: application, deploymentTemplateHistoryService: deploymentTemplateHistoryService, scopedVariableManager: scopedVariableManager, deployedAppMetricsService: deployedAppMetricsService, diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index 7a86482794..8e45fdbe51 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -23,6 +23,7 @@ import ( errors3 "errors" "fmt" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" "path" "strconv" "strings" @@ -42,10 +43,8 @@ import ( application2 "github.com/devtron-labs/devtron/client/argocdServer/application" gitSensorClient "github.com/devtron-labs/devtron/client/gitSensor" "github.com/devtron-labs/devtron/internal/middleware" - app2 "github.com/devtron-labs/devtron/internal/sql/repository/app" bean4 "github.com/devtron-labs/devtron/pkg/app/bean" "github.com/devtron-labs/devtron/pkg/app/status" - "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" "github.com/devtron-labs/devtron/pkg/auth/user" "github.com/devtron-labs/devtron/pkg/chartRepo/repository" "github.com/devtron-labs/devtron/pkg/dockerRegistry" @@ -130,11 +129,8 @@ type WorkflowDagExecutorImpl struct { pipelineOverrideRepository chartConfig.PipelineOverrideRepository ciArtifactRepository repository.CiArtifactRepository user user.UserService - enforcer casbin.Enforcer enforcerUtil rbac.EnforcerUtil groupRepository repository.DeploymentGroupRepository - tokenCache *util3.TokenCache - acdAuthConfig *util3.ACDAuthConfig envRepository repository2.EnvironmentRepository eventFactory client.EventFactory eventClient client.EventClient @@ -150,15 +146,13 @@ type WorkflowDagExecutorImpl struct { appLabelRepository pipelineConfig.AppLabelRepository gitSensorGrpcClient gitSensorClient.Client k8sCommonService k8s.K8sCommonService - pipelineStageRepository repository4.PipelineStageRepository pipelineStageService PipelineStageService config *types.CdConfig appServiceConfig *app.AppServiceConfig globalPluginService plugin.GlobalPluginService - scopedVariableManager variables.ScopedVariableCMCSManager - variableSnapshotHistoryService variables.VariableSnapshotHistoryService - pluginInputVariableParser PluginInputVariableParser + scopedVariableManager variables.ScopedVariableCMCSManager + pluginInputVariableParser PluginInputVariableParser devtronAsyncHelmInstallRequestMap map[int]bool devtronAsyncHelmInstallRequestLock *sync.Mutex @@ -179,15 +173,12 @@ type WorkflowDagExecutorImpl struct { chartRepository chartRepoRepository.ChartRepository chartTemplateService util.ChartTemplateService strategyHistoryRepository repository3.PipelineStrategyHistoryRepository - appRepository app2.AppRepository deploymentTemplateHistoryRepository repository3.DeploymentTemplateHistoryRepository argoK8sClient argocdServer.ArgoK8sClient configMapRepository chartConfig.ConfigMapRepository configMapHistoryRepository repository3.ConfigMapHistoryRepository - refChartDir chartRepoRepository.RefChartDir helmAppService client2.HelmAppService helmAppClient client2.HelmAppClient - chartRefRepository chartRepoRepository.ChartRefRepository environmentConfigRepository chartConfig.EnvConfigOverrideRepository dbMigrationConfigRepository pipelineConfig.DbMigrationConfigRepository mergeUtil *util.MergeUtil @@ -195,10 +186,10 @@ type WorkflowDagExecutorImpl struct { gitFactory *util.GitFactory acdClient application2.ServiceClient argoClientWrapperService argocdServer.ArgoClientWrapperService - pipelineConfigListenerService PipelineConfigListenerService customTagService CustomTagService ACDConfig *argocdServer.ACDConfig deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService + chartRefService chartRef.ChartRefService } const kedaAutoscaling = "kedaAutoscaling" @@ -254,8 +245,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi user user.UserService, groupRepository repository.DeploymentGroupRepository, envRepository repository2.EnvironmentRepository, - enforcer casbin.Enforcer, enforcerUtil rbac.EnforcerUtil, tokenCache *util3.TokenCache, - acdAuthConfig *util3.ACDAuthConfig, eventFactory client.EventFactory, + enforcerUtil rbac.EnforcerUtil, eventFactory client.EventFactory, eventClient client.EventClient, cvePolicyRepository security.CvePolicyRepository, scanResultRepository security.ImageScanResultRepository, appWorkflowRepository appWorkflow.AppWorkflowRepository, @@ -267,7 +257,6 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi ciWorkflowRepository pipelineConfig.CiWorkflowRepository, appLabelRepository pipelineConfig.AppLabelRepository, gitSensorGrpcClient gitSensorClient.Client, pipelineStageService PipelineStageService, k8sCommonService k8s.K8sCommonService, - variableSnapshotHistoryService variables.VariableSnapshotHistoryService, globalPluginService plugin.GlobalPluginService, pluginInputVariableParser PluginInputVariableParser, scopedVariableManager variables.ScopedVariableCMCSManager, @@ -286,15 +275,12 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi chartRepository chartRepoRepository.ChartRepository, chartTemplateService util.ChartTemplateService, strategyHistoryRepository repository3.PipelineStrategyHistoryRepository, - appRepository app2.AppRepository, deploymentTemplateHistoryRepository repository3.DeploymentTemplateHistoryRepository, ArgoK8sClient argocdServer.ArgoK8sClient, configMapRepository chartConfig.ConfigMapRepository, configMapHistoryRepository repository3.ConfigMapHistoryRepository, - refChartDir chartRepoRepository.RefChartDir, helmAppService client2.HelmAppService, helmAppClient client2.HelmAppClient, - chartRefRepository chartRepoRepository.ChartRefRepository, environmentConfigRepository chartConfig.EnvConfigOverrideRepository, dbMigrationConfigRepository pipelineConfig.DbMigrationConfigRepository, mergeUtil *util.MergeUtil, @@ -305,7 +291,8 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi pipelineConfigListenerService PipelineConfigListenerService, customTagService CustomTagService, ACDConfig *argocdServer.ACDConfig, - deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService) *WorkflowDagExecutorImpl { + deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService, + chartRefService chartRef.ChartRefService) *WorkflowDagExecutorImpl { wde := &WorkflowDagExecutorImpl{logger: Logger, pipelineRepository: pipelineRepository, cdWorkflowRepository: cdWorkflowRepository, @@ -317,11 +304,8 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi materialRepository: materialRepository, pipelineOverrideRepository: pipelineOverrideRepository, user: user, - enforcer: enforcer, enforcerUtil: enforcerUtil, groupRepository: groupRepository, - tokenCache: tokenCache, - acdAuthConfig: acdAuthConfig, envRepository: envRepository, eventFactory: eventFactory, eventClient: eventClient, @@ -361,15 +345,12 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi chartRepository: chartRepository, chartTemplateService: chartTemplateService, strategyHistoryRepository: strategyHistoryRepository, - appRepository: appRepository, deploymentTemplateHistoryRepository: deploymentTemplateHistoryRepository, argoK8sClient: ArgoK8sClient, configMapRepository: configMapRepository, configMapHistoryRepository: configMapHistoryRepository, - refChartDir: refChartDir, helmAppService: helmAppService, helmAppClient: helmAppClient, - chartRefRepository: chartRefRepository, environmentConfigRepository: environmentConfigRepository, dbMigrationConfigRepository: dbMigrationConfigRepository, mergeUtil: mergeUtil, @@ -377,10 +358,10 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi gitFactory: gitFactory, acdClient: acdClient, argoClientWrapperService: argoClientWrapperService, - pipelineConfigListenerService: pipelineConfigListenerService, customTagService: customTagService, ACDConfig: ACDConfig, deployedAppMetricsService: deployedAppMetricsService, + chartRefService: chartRefService, } config, err := types.GetCdConfig() if err != nil { @@ -3471,7 +3452,7 @@ func (impl *WorkflowDagExecutorImpl) createHelmAppForCdPipeline(overrideRequest Name: pipeline.App.AppName, Version: envOverride.Chart.ChartVersion, } - referenceTemplatePath := path.Join(string(impl.refChartDir), envOverride.Chart.ReferenceTemplate) + referenceTemplatePath := path.Join(chartRepoRepository.RefChartDirPath, envOverride.Chart.ReferenceTemplate) if util.IsHelmApp(pipeline.DeploymentAppType) { referenceChartByte := envOverride.Chart.ReferenceChart @@ -3653,7 +3634,7 @@ func (impl *WorkflowDagExecutorImpl) GetEnvOverrideByTriggerType(overrideRequest } //getting chart_ref by id _, span = otel.Tracer("orchestrator").Start(ctx, "chartRefRepository.FindByVersionAndName") - chartRef, err := impl.chartRefRepository.FindByVersionAndName(templateName, templateVersion) + chartRefDto, err := impl.chartRefService.FindByVersionAndName(templateName, templateVersion) span.End() if err != nil { impl.logger.Errorw("error in getting chartRef by version and name", "err", err, "version", templateVersion, "name", templateName) @@ -3661,10 +3642,10 @@ func (impl *WorkflowDagExecutorImpl) GetEnvOverrideByTriggerType(overrideRequest } //assuming that if a chartVersion is deployed then it's envConfigOverride will be available _, span = otel.Tracer("orchestrator").Start(ctx, "environmentConfigRepository.GetByAppIdEnvIdAndChartRefId") - envOverride, err = impl.environmentConfigRepository.GetByAppIdEnvIdAndChartRefId(overrideRequest.AppId, overrideRequest.EnvId, chartRef.Id) + envOverride, err = impl.environmentConfigRepository.GetByAppIdEnvIdAndChartRefId(overrideRequest.AppId, overrideRequest.EnvId, chartRefDto.Id) span.End() if err != nil { - impl.logger.Errorw("error in getting envConfigOverride for pipeline for specific chartVersion", "err", err, "appId", overrideRequest.AppId, "envId", overrideRequest.EnvId, "chartRefId", chartRef.Id) + impl.logger.Errorw("error in getting envConfigOverride for pipeline for specific chartVersion", "err", err, "appId", overrideRequest.AppId, "envId", overrideRequest.EnvId, "chartRefId", chartRefDto.Id) return nil, err } diff --git a/pkg/pipeline/history/DeploymentTemplateHistoryService.go b/pkg/pipeline/history/DeploymentTemplateHistoryService.go index beeae5af84..8871f9cd98 100644 --- a/pkg/pipeline/history/DeploymentTemplateHistoryService.go +++ b/pkg/pipeline/history/DeploymentTemplateHistoryService.go @@ -3,6 +3,7 @@ package history import ( "context" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" "time" "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" @@ -38,28 +39,28 @@ type DeploymentTemplateHistoryServiceImpl struct { deploymentTemplateHistoryRepository repository.DeploymentTemplateHistoryRepository pipelineRepository pipelineConfig.PipelineRepository chartRepository chartRepoRepository.ChartRepository - chartRefRepository chartRepoRepository.ChartRefRepository userService user.UserService cdWorkflowRepository pipelineConfig.CdWorkflowRepository scopedVariableManager variables.ScopedVariableManager deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService + chartRefService chartRef.ChartRefService } func NewDeploymentTemplateHistoryServiceImpl(logger *zap.SugaredLogger, deploymentTemplateHistoryRepository repository.DeploymentTemplateHistoryRepository, pipelineRepository pipelineConfig.PipelineRepository, chartRepository chartRepoRepository.ChartRepository, - chartRefRepository chartRepoRepository.ChartRefRepository, userService user.UserService, - cdWorkflowRepository pipelineConfig.CdWorkflowRepository, scopedVariableManager variables.ScopedVariableManager, - deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService) *DeploymentTemplateHistoryServiceImpl { + userService user.UserService, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, + scopedVariableManager variables.ScopedVariableManager, deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService, + chartRefService chartRef.ChartRefService) *DeploymentTemplateHistoryServiceImpl { return &DeploymentTemplateHistoryServiceImpl{ logger: logger, deploymentTemplateHistoryRepository: deploymentTemplateHistoryRepository, pipelineRepository: pipelineRepository, chartRepository: chartRepository, - chartRefRepository: chartRefRepository, userService: userService, cdWorkflowRepository: cdWorkflowRepository, scopedVariableManager: scopedVariableManager, deployedAppMetricsService: deployedAppMetricsService, + chartRefService: chartRefService, } } @@ -70,22 +71,19 @@ func (impl DeploymentTemplateHistoryServiceImpl) CreateDeploymentTemplateHistory impl.logger.Errorw("err in getting pipelines, CreateDeploymentTemplateHistoryFromGlobalTemplate", "err", err, "chart", chart) return err } - chartRef, err := impl.chartRefRepository.FindById(chart.ChartRefId) + chartRefDto, err := impl.chartRefService.FindById(chart.ChartRefId) if err != nil { impl.logger.Errorw("err in getting chartRef, CreateDeploymentTemplateHistoryFromGlobalTemplate", "err", err, "chart", chart) return err } - if len(chartRef.Name) == 0 { - chartRef.Name = "Rollout Deployment" - } //creating history without pipeline id historyModel := &repository.DeploymentTemplateHistory{ AppId: chart.AppId, ImageDescriptorTemplate: chart.ImageDescriptorTemplate, Template: chart.GlobalOverride, Deployed: false, - TemplateName: chartRef.Name, - TemplateVersion: chartRef.Version, + TemplateName: chartRefDto.Name, + TemplateVersion: chartRefDto.Version, IsAppMetricsEnabled: IsAppMetricsEnabled, AuditLog: sql.AuditLog{ CreatedOn: chart.CreatedOn, @@ -111,8 +109,8 @@ func (impl DeploymentTemplateHistoryServiceImpl) CreateDeploymentTemplateHistory ImageDescriptorTemplate: chart.ImageDescriptorTemplate, Template: chart.GlobalOverride, Deployed: false, - TemplateName: chartRef.Name, - TemplateVersion: chartRef.Version, + TemplateName: chartRefDto.Name, + TemplateVersion: chartRefDto.Version, IsAppMetricsEnabled: IsAppMetricsEnabled, AuditLog: sql.AuditLog{ CreatedOn: chart.CreatedOn, @@ -141,14 +139,11 @@ func (impl DeploymentTemplateHistoryServiceImpl) CreateDeploymentTemplateHistory impl.logger.Errorw("err in getting global deployment template", "err", err, "chart", chart) return err } - chartRef, err := impl.chartRefRepository.FindById(chart.ChartRefId) + chartRefDto, err := impl.chartRefService.FindById(chart.ChartRefId) if err != nil { - impl.logger.Errorw("err in getting chartRef, CreateDeploymentTemplateHistoryFromGlobalTemplate", "err", err, "chartRef", chartRef) + impl.logger.Errorw("err in getting chartRef, CreateDeploymentTemplateHistoryFromGlobalTemplate", "err", err, "chartRef", chartRefDto) return err } - if len(chartRef.Name) == 0 { - chartRef.Name = "Rollout Deployment" - } if pipelineId == 0 { pipeline, err := impl.pipelineRepository.GetByEnvOverrideIdAndEnvId(envOverride.Id, envOverride.TargetEnvironment) if err != nil && err != pg.ErrNoRows { @@ -163,8 +158,8 @@ func (impl DeploymentTemplateHistoryServiceImpl) CreateDeploymentTemplateHistory ImageDescriptorTemplate: chart.ImageDescriptorTemplate, TargetEnvironment: envOverride.TargetEnvironment, Deployed: false, - TemplateName: chartRef.Name, - TemplateVersion: chartRef.Version, + TemplateName: chartRefDto.Name, + TemplateVersion: chartRefDto.Version, IsAppMetricsEnabled: IsAppMetricsEnabled, AuditLog: sql.AuditLog{ CreatedOn: envOverride.CreatedOn, @@ -193,14 +188,11 @@ func (impl DeploymentTemplateHistoryServiceImpl) CreateDeploymentTemplateHistory } func (impl DeploymentTemplateHistoryServiceImpl) CreateDeploymentTemplateHistoryForDeploymentTrigger(pipeline *pipelineConfig.Pipeline, envOverride *chartConfig.EnvConfigOverride, renderedImageTemplate string, deployedOn time.Time, deployedBy int32) (*repository.DeploymentTemplateHistory, error) { - chartRef, err := impl.chartRefRepository.FindById(envOverride.Chart.ChartRefId) + chartRefDto, err := impl.chartRefService.FindById(envOverride.Chart.ChartRefId) if err != nil { - impl.logger.Errorw("err in getting chartRef, CreateDeploymentTemplateHistoryFromGlobalTemplate", "err", err, "chartRef", chartRef) + impl.logger.Errorw("err in getting chartRef, CreateDeploymentTemplateHistoryFromGlobalTemplate", "err", err, "chartRef", chartRefDto) return nil, err } - if len(chartRef.Name) == 0 { - chartRef.Name = "Rollout Deployment" - } isAppMetricsEnabled, err := impl.deployedAppMetricsService.GetMetricsFlagForAPipelineByAppIdAndEnvId(pipeline.AppId, pipeline.EnvironmentId) if err != nil { impl.logger.Errorw("error, GetMetricsFlagForAPipelineByAppIdAndEnvId", "err", err, "appId", pipeline.AppId, "envId", pipeline.EnvironmentId) @@ -214,8 +206,8 @@ func (impl DeploymentTemplateHistoryServiceImpl) CreateDeploymentTemplateHistory Deployed: true, DeployedBy: deployedBy, DeployedOn: deployedOn, - TemplateName: chartRef.Name, - TemplateVersion: chartRef.Version, + TemplateName: chartRefDto.Name, + TemplateVersion: chartRefDto.Version, IsAppMetricsEnabled: isAppMetricsEnabled, AuditLog: sql.AuditLog{ CreatedOn: deployedOn, diff --git a/wire_gen.go b/wire_gen.go index d330b3674e..f32319b90d 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -45,7 +45,7 @@ import ( "github.com/devtron-labs/devtron/client/argocdServer/application" "github.com/devtron-labs/devtron/client/argocdServer/cluster" "github.com/devtron-labs/devtron/client/argocdServer/connection" - repository9 "github.com/devtron-labs/devtron/client/argocdServer/repository" + repository12 "github.com/devtron-labs/devtron/client/argocdServer/repository" "github.com/devtron-labs/devtron/client/cron" "github.com/devtron-labs/devtron/client/dashboard" "github.com/devtron-labs/devtron/client/events" @@ -74,18 +74,17 @@ import ( "github.com/devtron-labs/devtron/pkg/appClone" "github.com/devtron-labs/devtron/pkg/appClone/batch" appStatus2 "github.com/devtron-labs/devtron/pkg/appStatus" - "github.com/devtron-labs/devtron/pkg/appStore/bean" "github.com/devtron-labs/devtron/pkg/appStore/chartProvider" "github.com/devtron-labs/devtron/pkg/appStore/deployment/common" "github.com/devtron-labs/devtron/pkg/appStore/deployment/fullMode" repository3 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" - "github.com/devtron-labs/devtron/pkg/appStore/deployment/service" + service2 "github.com/devtron-labs/devtron/pkg/appStore/deployment/service" "github.com/devtron-labs/devtron/pkg/appStore/deployment/tool" "github.com/devtron-labs/devtron/pkg/appStore/deployment/tool/gitops" "github.com/devtron-labs/devtron/pkg/appStore/discover/repository" service3 "github.com/devtron-labs/devtron/pkg/appStore/discover/service" "github.com/devtron-labs/devtron/pkg/appStore/values/repository" - service2 "github.com/devtron-labs/devtron/pkg/appStore/values/service" + "github.com/devtron-labs/devtron/pkg/appStore/values/service" appWorkflow2 "github.com/devtron-labs/devtron/pkg/appWorkflow" "github.com/devtron-labs/devtron/pkg/attributes" "github.com/devtron-labs/devtron/pkg/auth/authentication" @@ -103,14 +102,15 @@ import ( "github.com/devtron-labs/devtron/pkg/commonService" delete2 "github.com/devtron-labs/devtron/pkg/delete" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" "github.com/devtron-labs/devtron/pkg/deploymentGroup" "github.com/devtron-labs/devtron/pkg/devtronResource" - repository8 "github.com/devtron-labs/devtron/pkg/devtronResource/repository" + repository7 "github.com/devtron-labs/devtron/pkg/devtronResource/repository" "github.com/devtron-labs/devtron/pkg/dockerRegistry" "github.com/devtron-labs/devtron/pkg/externalLink" "github.com/devtron-labs/devtron/pkg/generateManifest" "github.com/devtron-labs/devtron/pkg/genericNotes" - repository10 "github.com/devtron-labs/devtron/pkg/genericNotes/repository" + repository11 "github.com/devtron-labs/devtron/pkg/genericNotes/repository" "github.com/devtron-labs/devtron/pkg/git" "github.com/devtron-labs/devtron/pkg/gitops" jira2 "github.com/devtron-labs/devtron/pkg/jira" @@ -127,11 +127,11 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/history" - repository6 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" - repository11 "github.com/devtron-labs/devtron/pkg/pipeline/repository" + repository8 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" + repository9 "github.com/devtron-labs/devtron/pkg/pipeline/repository" "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/plugin" - repository12 "github.com/devtron-labs/devtron/pkg/plugin/repository" + repository10 "github.com/devtron-labs/devtron/pkg/plugin/repository" "github.com/devtron-labs/devtron/pkg/projectManagementService/jira" resourceGroup2 "github.com/devtron-labs/devtron/pkg/resourceGroup" "github.com/devtron-labs/devtron/pkg/resourceQualifiers" @@ -142,12 +142,12 @@ import ( "github.com/devtron-labs/devtron/pkg/sql" "github.com/devtron-labs/devtron/pkg/team" "github.com/devtron-labs/devtron/pkg/terminal" - util2 "github.com/devtron-labs/devtron/pkg/util" + util3 "github.com/devtron-labs/devtron/pkg/util" "github.com/devtron-labs/devtron/pkg/variables" "github.com/devtron-labs/devtron/pkg/variables/parsers" - repository7 "github.com/devtron-labs/devtron/pkg/variables/repository" + repository6 "github.com/devtron-labs/devtron/pkg/variables/repository" "github.com/devtron-labs/devtron/pkg/webhook/helm" - util3 "github.com/devtron-labs/devtron/util" + util2 "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/argo" "github.com/devtron-labs/devtron/util/rbac" ) @@ -178,9 +178,7 @@ func InitializeApp() (*App, error) { mergeUtil := &util.MergeUtil{ Logger: sugaredLogger, } - ciArtifactRepositoryImpl := repository.NewCiArtifactRepositoryImpl(db, sugaredLogger) pipelineRepositoryImpl := pipelineConfig.NewPipelineRepositoryImpl(db, sugaredLogger) - dbMigrationConfigRepositoryImpl := pipelineConfig.NewDbMigrationConfigRepositoryImpl(db, sugaredLogger) httpClient := util.NewHttpClient() eventClientConfig, err := client.GetEventClientConfig() if err != nil { @@ -292,143 +290,85 @@ func InitializeApp() (*App, error) { cdWorkflowRepositoryImpl := pipelineConfig.NewCdWorkflowRepositoryImpl(db, sugaredLogger) ciWorkflowRepositoryImpl := pipelineConfig.NewCiWorkflowRepositoryImpl(db, sugaredLogger) ciPipelineMaterialRepositoryImpl := pipelineConfig.NewCiPipelineMaterialRepositoryImpl(db, sugaredLogger) + ciArtifactRepositoryImpl := repository.NewCiArtifactRepositoryImpl(db, sugaredLogger) eventSimpleFactoryImpl := client.NewEventSimpleFactoryImpl(sugaredLogger, cdWorkflowRepositoryImpl, pipelineOverrideRepositoryImpl, ciWorkflowRepositoryImpl, ciPipelineMaterialRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, userRepositoryImpl, ciArtifactRepositoryImpl) applicationServiceClientImpl := application.NewApplicationClientImpl(sugaredLogger, argoCDConnectionManagerImpl) - acdAuthConfig, err := util2.GetACDAuthConfig() - if err != nil { - return nil, err - } - tokenCache := util2.NewTokenCache(sugaredLogger, acdAuthConfig, userAuthServiceImpl) - syncedEnforcer := casbin.Create() - enforcerImpl := casbin.NewEnforcerImpl(syncedEnforcer, sessionManager, sugaredLogger) - enforcerUtilImpl := rbac.NewEnforcerUtilImpl(sugaredLogger, teamRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, clusterRepositoryImpl, enforcerImpl) - appListingRepositoryQueryBuilder := helper.NewAppListingRepositoryQueryBuilder(sugaredLogger) - appListingRepositoryImpl := repository.NewAppListingRepositoryImpl(sugaredLogger, db, appListingRepositoryQueryBuilder, environmentRepositoryImpl) - pipelineConfigRepositoryImpl := chartConfig.NewPipelineConfigRepository(db) configMapRepositoryImpl := chartConfig.NewConfigMapRepositoryImpl(sugaredLogger, db) chartRepositoryImpl := chartRepoRepository.NewChartRepository(db) dockerArtifactStoreRepositoryImpl := repository5.NewDockerArtifactStoreRepositoryImpl(db) gitProviderRepositoryImpl := repository.NewGitProviderRepositoryImpl(db) commonServiceImpl := commonService.NewCommonServiceImpl(sugaredLogger, chartRepositoryImpl, envConfigOverrideRepositoryImpl, gitOpsConfigRepositoryImpl, dockerArtifactStoreRepositoryImpl, attributesRepositoryImpl, gitProviderRepositoryImpl, environmentRepositoryImpl, teamRepositoryImpl, appRepositoryImpl) - imageScanDeployInfoRepositoryImpl := security.NewImageScanDeployInfoRepositoryImpl(db, sugaredLogger) - imageScanHistoryRepositoryImpl := security.NewImageScanHistoryRepositoryImpl(db, sugaredLogger) - argoK8sClientImpl := argocdServer.NewArgoK8sClientImpl(sugaredLogger) gitCliUtil := util.NewGitCliUtil(sugaredLogger) gitFactory, err := util.NewGitFactory(sugaredLogger, gitOpsConfigRepositoryImpl, gitCliUtil) if err != nil { return nil, err } - pipelineStrategyHistoryRepositoryImpl := repository6.NewPipelineStrategyHistoryRepositoryImpl(sugaredLogger, db) - pipelineStrategyHistoryServiceImpl := history.NewPipelineStrategyHistoryServiceImpl(sugaredLogger, pipelineStrategyHistoryRepositoryImpl, userServiceImpl) - configMapHistoryRepositoryImpl := repository6.NewConfigMapHistoryRepositoryImpl(sugaredLogger, db) - scopedVariableRepositoryImpl := repository7.NewScopedVariableRepository(db, sugaredLogger) - qualifiersMappingRepositoryImpl, err := resourceQualifiers.NewQualifiersMappingRepositoryImpl(db, sugaredLogger) + globalEnvVariables, err := util2.GetGlobalEnvVariables() if err != nil { return nil, err } - devtronResourceSearchableKeyRepositoryImpl := repository8.NewDevtronResourceSearchableKeyRepositoryImpl(sugaredLogger, db) - devtronResourceSearchableKeyServiceImpl, err := devtronResource.NewDevtronResourceSearchableKeyServiceImpl(sugaredLogger, devtronResourceSearchableKeyRepositoryImpl) - if err != nil { - return nil, err - } - qualifierMappingServiceImpl, err := resourceQualifiers.NewQualifierMappingServiceImpl(sugaredLogger, qualifiersMappingRepositoryImpl, devtronResourceSearchableKeyServiceImpl) + chartTemplateServiceImpl := util.NewChartTemplateServiceImpl(sugaredLogger, gitFactory, globalEnvVariables, gitOpsConfigRepositoryImpl, userRepositoryImpl, chartRepositoryImpl) + devtronSecretConfig, err := util2.GetDevtronSecretName() if err != nil { return nil, err } - scopedVariableServiceImpl, err := variables.NewScopedVariableServiceImpl(sugaredLogger, scopedVariableRepositoryImpl, qualifierMappingServiceImpl) + versionServiceImpl := argocdServer.NewVersionServiceImpl(sugaredLogger, argoCDConnectionManagerImpl) + argoUserServiceImpl, err := argo.NewArgoUserServiceImpl(sugaredLogger, clusterServiceImplExtended, devtronSecretConfig, runtimeConfig, gitOpsConfigRepositoryImpl, argoCDConnectionManagerImpl, versionServiceImpl, k8sUtil) if err != nil { return nil, err } - variableEntityMappingRepositoryImpl := repository7.NewVariableEntityMappingRepository(sugaredLogger, db) - variableEntityMappingServiceImpl := variables.NewVariableEntityMappingServiceImpl(variableEntityMappingRepositoryImpl, sugaredLogger) - variableSnapshotHistoryRepositoryImpl := repository7.NewVariableSnapshotHistoryRepository(sugaredLogger, db) - variableSnapshotHistoryServiceImpl := variables.NewVariableSnapshotHistoryServiceImpl(variableSnapshotHistoryRepositoryImpl, sugaredLogger) - variableTemplateParserImpl, err := parsers.NewVariableTemplateParserImpl(sugaredLogger) + pipelineStatusTimelineRepositoryImpl := pipelineConfig.NewPipelineStatusTimelineRepositoryImpl(db, sugaredLogger) + pipelineStatusTimelineResourcesRepositoryImpl := pipelineConfig.NewPipelineStatusTimelineResourcesRepositoryImpl(db, sugaredLogger) + pipelineStatusTimelineResourcesServiceImpl := status.NewPipelineStatusTimelineResourcesServiceImpl(db, sugaredLogger, pipelineStatusTimelineResourcesRepositoryImpl) + pipelineStatusSyncDetailRepositoryImpl := pipelineConfig.NewPipelineStatusSyncDetailRepositoryImpl(db, sugaredLogger) + pipelineStatusSyncDetailServiceImpl := status.NewPipelineStatusSyncDetailServiceImpl(sugaredLogger, pipelineStatusSyncDetailRepositoryImpl) + installedAppVersionHistoryRepositoryImpl := repository3.NewInstalledAppVersionHistoryRepositoryImpl(sugaredLogger, db) + pipelineStatusTimelineServiceImpl := status.NewPipelineStatusTimelineServiceImpl(sugaredLogger, pipelineStatusTimelineRepositoryImpl, cdWorkflowRepositoryImpl, userServiceImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl) + appServiceConfig, err := app2.GetAppServiceConfig() if err != nil { return nil, err } - scopedVariableCMCSManagerImpl, err := variables.NewScopedVariableCMCSManagerImpl(sugaredLogger, scopedVariableServiceImpl, variableEntityMappingServiceImpl, variableSnapshotHistoryServiceImpl, variableTemplateParserImpl) + syncedEnforcer := casbin.Create() + enforcerImpl := casbin.NewEnforcerImpl(syncedEnforcer, sessionManager, sugaredLogger) + enforcerUtilImpl := rbac.NewEnforcerUtilImpl(sugaredLogger, teamRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, clusterRepositoryImpl, enforcerImpl) + appStatusServiceImpl := appStatus2.NewAppStatusServiceImpl(appStatusRepositoryImpl, sugaredLogger, enforcerImpl, enforcerUtilImpl) + scopedVariableRepositoryImpl := repository6.NewScopedVariableRepository(db, sugaredLogger) + qualifiersMappingRepositoryImpl, err := resourceQualifiers.NewQualifiersMappingRepositoryImpl(db, sugaredLogger) if err != nil { return nil, err } - configMapHistoryServiceImpl := history.NewConfigMapHistoryServiceImpl(sugaredLogger, configMapHistoryRepositoryImpl, pipelineRepositoryImpl, configMapRepositoryImpl, userServiceImpl, scopedVariableCMCSManagerImpl) - deploymentTemplateHistoryRepositoryImpl := repository6.NewDeploymentTemplateHistoryRepositoryImpl(sugaredLogger, db) - chartRefRepositoryImpl := chartRepoRepository.NewChartRefRepositoryImpl(db) - scopedVariableManagerImpl, err := variables.NewScopedVariableManagerImpl(sugaredLogger, scopedVariableServiceImpl, variableEntityMappingServiceImpl, variableSnapshotHistoryServiceImpl, variableTemplateParserImpl) + devtronResourceSearchableKeyRepositoryImpl := repository7.NewDevtronResourceSearchableKeyRepositoryImpl(sugaredLogger, db) + devtronResourceSearchableKeyServiceImpl, err := devtronResource.NewDevtronResourceSearchableKeyServiceImpl(sugaredLogger, devtronResourceSearchableKeyRepositoryImpl) if err != nil { return nil, err } - appLevelMetricsRepositoryImpl := repository.NewAppLevelMetricsRepositoryImpl(db, sugaredLogger) - envLevelAppMetricsRepositoryImpl := repository.NewEnvLevelAppMetricsRepositoryImpl(db, sugaredLogger) - deployedAppMetricsServiceImpl := deployedAppMetrics.NewDeployedAppMetricsServiceImpl(sugaredLogger, chartRefRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl) - deploymentTemplateHistoryServiceImpl := history.NewDeploymentTemplateHistoryServiceImpl(sugaredLogger, deploymentTemplateHistoryRepositoryImpl, pipelineRepositoryImpl, chartRepositoryImpl, chartRefRepositoryImpl, userServiceImpl, cdWorkflowRepositoryImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl) - chartWorkingDir := _wireChartWorkingDirValue - globalEnvVariables, err := util3.GetGlobalEnvVariables() + qualifierMappingServiceImpl, err := resourceQualifiers.NewQualifierMappingServiceImpl(sugaredLogger, qualifiersMappingRepositoryImpl, devtronResourceSearchableKeyServiceImpl) if err != nil { return nil, err } - chartTemplateServiceImpl := util.NewChartTemplateServiceImpl(sugaredLogger, chartWorkingDir, httpClient, gitFactory, globalEnvVariables, gitOpsConfigRepositoryImpl, userRepositoryImpl, chartRepositoryImpl) - refChartDir := _wireRefChartDirValue - chartRepoRepositoryImpl := chartRepoRepository.NewChartRepoRepositoryImpl(db) - defaultChart := _wireDefaultChartValue - utilMergeUtil := util.MergeUtil{ - Logger: sugaredLogger, - } - repositoryServiceClientImpl := repository9.NewServiceClientImpl(sugaredLogger, argoCDConnectionManagerImpl) - chartServiceImpl := chart.NewChartServiceImpl(chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, appRepositoryImpl, refChartDir, defaultChart, utilMergeUtil, repositoryServiceClientImpl, chartRefRepositoryImpl, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, httpClient, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl) - devtronSecretConfig, err := util3.GetDevtronSecretName() + scopedVariableServiceImpl, err := variables.NewScopedVariableServiceImpl(sugaredLogger, scopedVariableRepositoryImpl, qualifierMappingServiceImpl) if err != nil { return nil, err } - versionServiceImpl := argocdServer.NewVersionServiceImpl(sugaredLogger, argoCDConnectionManagerImpl) - argoUserServiceImpl, err := argo.NewArgoUserServiceImpl(sugaredLogger, clusterServiceImplExtended, devtronSecretConfig, runtimeConfig, gitOpsConfigRepositoryImpl, argoCDConnectionManagerImpl, versionServiceImpl, k8sUtil) + variableEntityMappingRepositoryImpl := repository6.NewVariableEntityMappingRepository(sugaredLogger, db) + variableEntityMappingServiceImpl := variables.NewVariableEntityMappingServiceImpl(variableEntityMappingRepositoryImpl, sugaredLogger) + variableSnapshotHistoryRepositoryImpl := repository6.NewVariableSnapshotHistoryRepository(sugaredLogger, db) + variableSnapshotHistoryServiceImpl := variables.NewVariableSnapshotHistoryServiceImpl(variableSnapshotHistoryRepositoryImpl, sugaredLogger) + variableTemplateParserImpl, err := parsers.NewVariableTemplateParserImpl(sugaredLogger) if err != nil { return nil, err } - pipelineStatusTimelineRepositoryImpl := pipelineConfig.NewPipelineStatusTimelineRepositoryImpl(db, sugaredLogger) - appLabelRepositoryImpl := pipelineConfig.NewAppLabelRepositoryImpl(db) - genericNoteRepositoryImpl := repository10.NewGenericNoteRepositoryImpl(db) - genericNoteHistoryRepositoryImpl := repository10.NewGenericNoteHistoryRepositoryImpl(db) - genericNoteHistoryServiceImpl := genericNotes.NewGenericNoteHistoryServiceImpl(genericNoteHistoryRepositoryImpl, sugaredLogger) - genericNoteServiceImpl := genericNotes.NewGenericNoteServiceImpl(genericNoteRepositoryImpl, genericNoteHistoryServiceImpl, userRepositoryImpl, sugaredLogger) - materialRepositoryImpl := pipelineConfig.NewMaterialRepositoryImpl(db) - appCrudOperationServiceImpl := app2.NewAppCrudOperationServiceImpl(appLabelRepositoryImpl, sugaredLogger, appRepositoryImpl, userRepositoryImpl, installedAppRepositoryImpl, genericNoteServiceImpl, materialRepositoryImpl) - dockerRegistryIpsConfigRepositoryImpl := repository5.NewDockerRegistryIpsConfigRepositoryImpl(db) - ciTemplateOverrideRepositoryImpl := pipelineConfig.NewCiTemplateOverrideRepositoryImpl(db, sugaredLogger) - dockerRegistryIpsConfigServiceImpl := dockerRegistry.NewDockerRegistryIpsConfigServiceImpl(sugaredLogger, dockerRegistryIpsConfigRepositoryImpl, k8sUtil, clusterServiceImplExtended, ciPipelineRepositoryImpl, dockerArtifactStoreRepositoryImpl, ciTemplateOverrideRepositoryImpl) - pipelineStatusTimelineResourcesRepositoryImpl := pipelineConfig.NewPipelineStatusTimelineResourcesRepositoryImpl(db, sugaredLogger) - pipelineStatusTimelineResourcesServiceImpl := status.NewPipelineStatusTimelineResourcesServiceImpl(db, sugaredLogger, pipelineStatusTimelineResourcesRepositoryImpl) - pipelineStatusSyncDetailRepositoryImpl := pipelineConfig.NewPipelineStatusSyncDetailRepositoryImpl(db, sugaredLogger) - pipelineStatusSyncDetailServiceImpl := status.NewPipelineStatusSyncDetailServiceImpl(sugaredLogger, pipelineStatusSyncDetailRepositoryImpl) - installedAppVersionHistoryRepositoryImpl := repository3.NewInstalledAppVersionHistoryRepositoryImpl(sugaredLogger, db) - pipelineStatusTimelineServiceImpl := status.NewPipelineStatusTimelineServiceImpl(sugaredLogger, pipelineStatusTimelineRepositoryImpl, cdWorkflowRepositoryImpl, userServiceImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl) - appServiceConfig, err := app2.GetAppServiceConfig() + scopedVariableCMCSManagerImpl, err := variables.NewScopedVariableCMCSManagerImpl(sugaredLogger, scopedVariableServiceImpl, variableEntityMappingServiceImpl, variableSnapshotHistoryServiceImpl, variableTemplateParserImpl) if err != nil { return nil, err } - appStatusServiceImpl := appStatus2.NewAppStatusServiceImpl(appStatusRepositoryImpl, sugaredLogger, enforcerImpl, enforcerUtilImpl) - chartGroupDeploymentRepositoryImpl := repository3.NewChartGroupDeploymentRepositoryImpl(db, sugaredLogger) - clusterInstalledAppsRepositoryImpl := repository3.NewClusterInstalledAppsRepositoryImpl(db, sugaredLogger) - refChartProxyDir := _wireRefChartProxyDirValue - appStoreDeploymentCommonServiceImpl := appStoreDeploymentCommon.NewAppStoreDeploymentCommonServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, chartTemplateServiceImpl, refChartProxyDir, gitFactory, gitOpsConfigRepositoryImpl) - ociRegistryConfigRepositoryImpl := repository5.NewOCIRegistryConfigRepositoryImpl(db) - appStoreDeploymentHelmServiceImpl := appStoreDeploymentTool.NewAppStoreDeploymentHelmServiceImpl(sugaredLogger, helmAppServiceImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, helmAppClientImpl, installedAppRepositoryImpl, appStoreDeploymentCommonServiceImpl, ociRegistryConfigRepositoryImpl) acdConfig, err := argocdServer.GetACDDeploymentConfig() if err != nil { return nil, err } - argoClientWrapperServiceImpl := argocdServer.NewArgoClientWrapperServiceImpl(sugaredLogger, applicationServiceClientImpl, acdConfig, repositoryServiceClientImpl) - appStoreDeploymentFullModeServiceImpl := appStoreDeploymentFullMode.NewAppStoreDeploymentFullModeServiceImpl(sugaredLogger, chartTemplateServiceImpl, refChartProxyDir, repositoryServiceClientImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, applicationServiceClientImpl, argoK8sClientImpl, gitFactory, acdAuthConfig, globalEnvVariables, installedAppRepositoryImpl, tokenCache, argoUserServiceImpl, gitOpsConfigRepositoryImpl, pipelineStatusTimelineServiceImpl, appStoreDeploymentCommonServiceImpl, argoClientWrapperServiceImpl, pubSubClientServiceImpl, installedAppVersionHistoryRepositoryImpl, acdConfig) - appStoreDeploymentArgoCdServiceImpl := appStoreDeploymentGitopsTool.NewAppStoreDeploymentArgoCdServiceImpl(sugaredLogger, appStoreDeploymentFullModeServiceImpl, applicationServiceClientImpl, chartGroupDeploymentRepositoryImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, chartTemplateServiceImpl, gitFactory, argoUserServiceImpl, appStoreDeploymentCommonServiceImpl, helmAppServiceImpl, gitOpsConfigRepositoryImpl, appStatusServiceImpl, pipelineStatusTimelineServiceImpl, userServiceImpl, pipelineStatusTimelineRepositoryImpl, appStoreApplicationVersionRepositoryImpl, argoClientWrapperServiceImpl, acdConfig) - deploymentServiceTypeConfig, err := service.GetDeploymentServiceTypeConfig() - if err != nil { - return nil, err - } - appStoreDeploymentServiceImpl := service.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, clusterInstalledAppsRepositoryImpl, appRepositoryImpl, appStoreDeploymentHelmServiceImpl, appStoreDeploymentArgoCdServiceImpl, environmentServiceImpl, clusterServiceImplExtended, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, globalEnvVariables, installedAppVersionHistoryRepositoryImpl, gitOpsConfigRepositoryImpl, attributesServiceImpl, deploymentServiceTypeConfig, chartTemplateServiceImpl, acdConfig) - k8sCommonServiceImpl := k8s2.NewK8sCommonServiceImpl(sugaredLogger, k8sUtil, clusterServiceImplExtended) - manifestPushConfigRepositoryImpl := repository11.NewManifestPushConfigRepository(sugaredLogger, db) - gitOpsManifestPushServiceImpl := app2.NewGitOpsManifestPushServiceImpl(sugaredLogger, chartTemplateServiceImpl, chartServiceImpl, gitOpsConfigRepositoryImpl, gitFactory, pipelineStatusTimelineServiceImpl, pipelineStatusTimelineRepositoryImpl, acdConfig) - appServiceImpl := app2.NewAppService(envConfigOverrideRepositoryImpl, pipelineOverrideRepositoryImpl, mergeUtil, sugaredLogger, ciArtifactRepositoryImpl, pipelineRepositoryImpl, dbMigrationConfigRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, applicationServiceClientImpl, tokenCache, acdAuthConfig, enforcerImpl, enforcerUtilImpl, userServiceImpl, appListingRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, imageScanDeployInfoRepositoryImpl, imageScanHistoryRepositoryImpl, argoK8sClientImpl, gitFactory, pipelineStrategyHistoryServiceImpl, configMapHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, chartTemplateServiceImpl, refChartDir, chartRefRepositoryImpl, chartServiceImpl, helmAppClientImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, appCrudOperationServiceImpl, configMapHistoryRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, dockerRegistryIpsConfigServiceImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, gitOpsConfigRepositoryImpl, appStatusServiceImpl, installedAppRepositoryImpl, appStoreDeploymentServiceImpl, k8sCommonServiceImpl, installedAppVersionHistoryRepositoryImpl, globalEnvVariables, helmAppServiceImpl, manifestPushConfigRepositoryImpl, gitOpsManifestPushServiceImpl, argoClientWrapperServiceImpl, scopedVariableCMCSManagerImpl, acdConfig) + chartRefRepositoryImpl := chartRepoRepository.NewChartRefRepositoryImpl(db) + chartRefServiceImpl := chartRef.NewChartRefServiceImpl(sugaredLogger, chartRefRepositoryImpl, chartTemplateServiceImpl) + appServiceImpl := app2.NewAppService(envConfigOverrideRepositoryImpl, pipelineOverrideRepositoryImpl, mergeUtil, sugaredLogger, pipelineRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, applicationServiceClientImpl, appRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, chartTemplateServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, appStatusServiceImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, globalEnvVariables, scopedVariableCMCSManagerImpl, acdConfig, chartRefServiceImpl) validate, err := util.IntValidator() if err != nil { return nil, err @@ -441,17 +381,22 @@ func InitializeApp() (*App, error) { globalCMCSServiceImpl := pipeline.NewGlobalCMCSServiceImpl(sugaredLogger, globalCMCSRepositoryImpl) argoWorkflowExecutorImpl := executors.NewArgoWorkflowExecutorImpl(sugaredLogger) systemWorkflowExecutorImpl := executors.NewSystemWorkflowExecutorImpl(sugaredLogger, k8sUtil) + k8sCommonServiceImpl := k8s2.NewK8sCommonServiceImpl(sugaredLogger, k8sUtil, clusterServiceImplExtended) workflowServiceImpl, err := pipeline.NewWorkflowServiceImpl(sugaredLogger, environmentRepositoryImpl, ciCdConfig, appServiceImpl, globalCMCSServiceImpl, argoWorkflowExecutorImpl, k8sUtil, systemWorkflowExecutorImpl, k8sCommonServiceImpl) if err != nil { return nil, err } + materialRepositoryImpl := pipelineConfig.NewMaterialRepositoryImpl(db) deploymentGroupRepositoryImpl := repository.NewDeploymentGroupRepositoryImpl(sugaredLogger, db) cvePolicyRepositoryImpl := security.NewPolicyRepositoryImpl(db) imageScanResultRepositoryImpl := security.NewImageScanResultRepositoryImpl(db, sugaredLogger) appWorkflowRepositoryImpl := appWorkflow.NewAppWorkflowRepositoryImpl(sugaredLogger, db) - prePostCdScriptHistoryRepositoryImpl := repository6.NewPrePostCdScriptHistoryRepositoryImpl(sugaredLogger, db) + prePostCdScriptHistoryRepositoryImpl := repository8.NewPrePostCdScriptHistoryRepositoryImpl(sugaredLogger, db) + configMapHistoryRepositoryImpl := repository8.NewConfigMapHistoryRepositoryImpl(sugaredLogger, db) + configMapHistoryServiceImpl := history.NewConfigMapHistoryServiceImpl(sugaredLogger, configMapHistoryRepositoryImpl, pipelineRepositoryImpl, configMapRepositoryImpl, userServiceImpl, scopedVariableCMCSManagerImpl) prePostCdScriptHistoryServiceImpl := history.NewPrePostCdScriptHistoryServiceImpl(sugaredLogger, prePostCdScriptHistoryRepositoryImpl, configMapRepositoryImpl, configMapHistoryServiceImpl) ciTemplateRepositoryImpl := pipelineConfig.NewCiTemplateRepositoryImpl(db, sugaredLogger) + appLabelRepositoryImpl := pipelineConfig.NewAppLabelRepositoryImpl(db) clientConfig, err := gitSensor.GetConfig() if err != nil { return nil, err @@ -460,38 +405,73 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - pipelineStageRepositoryImpl := repository11.NewPipelineStageRepository(sugaredLogger, db) - globalPluginRepositoryImpl := repository12.NewGlobalPluginRepository(sugaredLogger, db) + pipelineStageRepositoryImpl := repository9.NewPipelineStageRepository(sugaredLogger, db) + globalPluginRepositoryImpl := repository10.NewGlobalPluginRepository(sugaredLogger, db) + scopedVariableManagerImpl, err := variables.NewScopedVariableManagerImpl(sugaredLogger, scopedVariableServiceImpl, variableEntityMappingServiceImpl, variableSnapshotHistoryServiceImpl, variableTemplateParserImpl) + if err != nil { + return nil, err + } pipelineStageServiceImpl := pipeline.NewPipelineStageService(sugaredLogger, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, pipelineRepositoryImpl, scopedVariableManagerImpl) globalPluginServiceImpl := plugin.NewGlobalPluginService(sugaredLogger, globalPluginRepositoryImpl, pipelineStageRepositoryImpl) + dockerRegistryIpsConfigRepositoryImpl := repository5.NewDockerRegistryIpsConfigRepositoryImpl(db) + ociRegistryConfigRepositoryImpl := repository5.NewOCIRegistryConfigRepositoryImpl(db) dockerRegistryConfigImpl := pipeline.NewDockerRegistryConfigImpl(sugaredLogger, helmAppServiceImpl, dockerArtifactStoreRepositoryImpl, dockerRegistryIpsConfigRepositoryImpl, ociRegistryConfigRepositoryImpl) imageTagRepositoryImpl := repository.NewImageTagRepository(db, sugaredLogger) customTagServiceImpl := pipeline.NewCustomTagService(sugaredLogger, imageTagRepositoryImpl) pluginInputVariableParserImpl := pipeline.NewPluginInputVariableParserImpl(sugaredLogger, dockerRegistryConfigImpl, customTagServiceImpl) + deploymentTemplateHistoryRepositoryImpl := repository8.NewDeploymentTemplateHistoryRepositoryImpl(sugaredLogger, db) + appLevelMetricsRepositoryImpl := repository.NewAppLevelMetricsRepositoryImpl(db, sugaredLogger) + envLevelAppMetricsRepositoryImpl := repository.NewEnvLevelAppMetricsRepositoryImpl(db, sugaredLogger) + deployedAppMetricsServiceImpl := deployedAppMetrics.NewDeployedAppMetricsServiceImpl(sugaredLogger, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, chartRefServiceImpl) + deploymentTemplateHistoryServiceImpl := history.NewDeploymentTemplateHistoryServiceImpl(sugaredLogger, deploymentTemplateHistoryRepositoryImpl, pipelineRepositoryImpl, chartRepositoryImpl, userServiceImpl, cdWorkflowRepositoryImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) + pipelineStrategyHistoryRepositoryImpl := repository8.NewPipelineStrategyHistoryRepositoryImpl(sugaredLogger, db) + pipelineStrategyHistoryServiceImpl := history.NewPipelineStrategyHistoryServiceImpl(sugaredLogger, pipelineStrategyHistoryRepositoryImpl, userServiceImpl) + manifestPushConfigRepositoryImpl := repository9.NewManifestPushConfigRepository(sugaredLogger, db) + gitOpsManifestPushServiceImpl := app2.NewGitOpsManifestPushServiceImpl(sugaredLogger, chartTemplateServiceImpl, gitOpsConfigRepositoryImpl, gitFactory, pipelineStatusTimelineServiceImpl, pipelineStatusTimelineRepositoryImpl, acdConfig, chartRefServiceImpl) + imageScanHistoryRepositoryImpl := security.NewImageScanHistoryRepositoryImpl(db, sugaredLogger) + imageScanDeployInfoRepositoryImpl := security.NewImageScanDeployInfoRepositoryImpl(db, sugaredLogger) + genericNoteRepositoryImpl := repository11.NewGenericNoteRepositoryImpl(db) + genericNoteHistoryRepositoryImpl := repository11.NewGenericNoteHistoryRepositoryImpl(db) + genericNoteHistoryServiceImpl := genericNotes.NewGenericNoteHistoryServiceImpl(genericNoteHistoryRepositoryImpl, sugaredLogger) + genericNoteServiceImpl := genericNotes.NewGenericNoteServiceImpl(genericNoteRepositoryImpl, genericNoteHistoryServiceImpl, userRepositoryImpl, sugaredLogger) + appCrudOperationServiceImpl := app2.NewAppCrudOperationServiceImpl(appLabelRepositoryImpl, sugaredLogger, appRepositoryImpl, userRepositoryImpl, installedAppRepositoryImpl, genericNoteServiceImpl, materialRepositoryImpl) + pipelineConfigRepositoryImpl := chartConfig.NewPipelineConfigRepository(db) + ciTemplateOverrideRepositoryImpl := pipelineConfig.NewCiTemplateOverrideRepositoryImpl(db, sugaredLogger) + dockerRegistryIpsConfigServiceImpl := dockerRegistry.NewDockerRegistryIpsConfigServiceImpl(sugaredLogger, dockerRegistryIpsConfigRepositoryImpl, k8sUtil, clusterServiceImplExtended, ciPipelineRepositoryImpl, dockerArtifactStoreRepositoryImpl, ciTemplateOverrideRepositoryImpl) + argoK8sClientImpl := argocdServer.NewArgoK8sClientImpl(sugaredLogger) + dbMigrationConfigRepositoryImpl := pipelineConfig.NewDbMigrationConfigRepositoryImpl(db, sugaredLogger) + repositoryServiceClientImpl := repository12.NewServiceClientImpl(sugaredLogger, argoCDConnectionManagerImpl) + argoClientWrapperServiceImpl := argocdServer.NewArgoClientWrapperServiceImpl(sugaredLogger, applicationServiceClientImpl, acdConfig, repositoryServiceClientImpl) pipelineConfigListenerServiceImpl := pipeline.NewPipelineConfigListenerServiceImpl(sugaredLogger) - workflowDagExecutorImpl := pipeline.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, pubSubClientServiceImpl, appServiceImpl, workflowServiceImpl, ciArtifactRepositoryImpl, ciPipelineRepositoryImpl, materialRepositoryImpl, pipelineOverrideRepositoryImpl, userServiceImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, enforcerImpl, enforcerUtilImpl, tokenCache, acdAuthConfig, eventSimpleFactoryImpl, eventRESTClientImpl, cvePolicyRepositoryImpl, imageScanResultRepositoryImpl, appWorkflowRepositoryImpl, prePostCdScriptHistoryServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineServiceImpl, ciTemplateRepositoryImpl, ciWorkflowRepositoryImpl, appLabelRepositoryImpl, clientImpl, pipelineStageServiceImpl, k8sCommonServiceImpl, variableSnapshotHistoryServiceImpl, globalPluginServiceImpl, pluginInputVariableParserImpl, scopedVariableCMCSManagerImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineStrategyHistoryServiceImpl, manifestPushConfigRepositoryImpl, gitOpsManifestPushServiceImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, appCrudOperationServiceImpl, pipelineConfigRepositoryImpl, dockerRegistryIpsConfigServiceImpl, chartRepositoryImpl, chartTemplateServiceImpl, pipelineStrategyHistoryRepositoryImpl, appRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, argoK8sClientImpl, configMapRepositoryImpl, configMapHistoryRepositoryImpl, refChartDir, helmAppServiceImpl, helmAppClientImpl, chartRefRepositoryImpl, envConfigOverrideRepositoryImpl, dbMigrationConfigRepositoryImpl, mergeUtil, gitOpsConfigRepositoryImpl, gitFactory, applicationServiceClientImpl, argoClientWrapperServiceImpl, pipelineConfigListenerServiceImpl, customTagServiceImpl, acdConfig, deployedAppMetricsServiceImpl) + workflowDagExecutorImpl := pipeline.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, pubSubClientServiceImpl, appServiceImpl, workflowServiceImpl, ciArtifactRepositoryImpl, ciPipelineRepositoryImpl, materialRepositoryImpl, pipelineOverrideRepositoryImpl, userServiceImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, enforcerUtilImpl, eventSimpleFactoryImpl, eventRESTClientImpl, cvePolicyRepositoryImpl, imageScanResultRepositoryImpl, appWorkflowRepositoryImpl, prePostCdScriptHistoryServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineServiceImpl, ciTemplateRepositoryImpl, ciWorkflowRepositoryImpl, appLabelRepositoryImpl, clientImpl, pipelineStageServiceImpl, k8sCommonServiceImpl, globalPluginServiceImpl, pluginInputVariableParserImpl, scopedVariableCMCSManagerImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineStrategyHistoryServiceImpl, manifestPushConfigRepositoryImpl, gitOpsManifestPushServiceImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, appCrudOperationServiceImpl, pipelineConfigRepositoryImpl, dockerRegistryIpsConfigServiceImpl, chartRepositoryImpl, chartTemplateServiceImpl, pipelineStrategyHistoryRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, argoK8sClientImpl, configMapRepositoryImpl, configMapHistoryRepositoryImpl, helmAppServiceImpl, helmAppClientImpl, envConfigOverrideRepositoryImpl, dbMigrationConfigRepositoryImpl, mergeUtil, gitOpsConfigRepositoryImpl, gitFactory, applicationServiceClientImpl, argoClientWrapperServiceImpl, pipelineConfigListenerServiceImpl, customTagServiceImpl, acdConfig, deployedAppMetricsServiceImpl, chartRefServiceImpl) deploymentGroupAppRepositoryImpl := repository.NewDeploymentGroupAppRepositoryImpl(sugaredLogger, db) deploymentGroupServiceImpl := deploymentGroup.NewDeploymentGroupServiceImpl(appRepositoryImpl, sugaredLogger, pipelineRepositoryImpl, ciPipelineRepositoryImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, deploymentGroupAppRepositoryImpl, ciArtifactRepositoryImpl, appWorkflowRepositoryImpl, workflowDagExecutorImpl) - deploymentConfigServiceImpl := pipeline.NewDeploymentConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, pipelineRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, configMapHistoryServiceImpl, chartRefRepositoryImpl, scopedVariableCMCSManagerImpl, deployedAppMetricsServiceImpl) + deploymentConfigServiceImpl := pipeline.NewDeploymentConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, pipelineRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, configMapHistoryServiceImpl, scopedVariableCMCSManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) pipelineTriggerRestHandlerImpl := restHandler.NewPipelineRestHandler(appServiceImpl, userServiceImpl, validate, enforcerImpl, teamServiceImpl, sugaredLogger, enforcerUtilImpl, workflowDagExecutorImpl, deploymentGroupServiceImpl, argoUserServiceImpl, deploymentConfigServiceImpl) sseSSE := sse.NewSSE() pipelineTriggerRouterImpl := router.NewPipelineTriggerRouter(pipelineTriggerRestHandlerImpl, sseSSE) - prePostCiScriptHistoryRepositoryImpl := repository6.NewPrePostCiScriptHistoryRepositoryImpl(sugaredLogger, db) + appListingRepositoryQueryBuilder := helper.NewAppListingRepositoryQueryBuilder(sugaredLogger) + appListingRepositoryImpl := repository.NewAppListingRepositoryImpl(sugaredLogger, db, appListingRepositoryQueryBuilder, environmentRepositoryImpl) + prePostCiScriptHistoryRepositoryImpl := repository8.NewPrePostCiScriptHistoryRepositoryImpl(sugaredLogger, db) prePostCiScriptHistoryServiceImpl := history.NewPrePostCiScriptHistoryServiceImpl(sugaredLogger, prePostCiScriptHistoryRepositoryImpl) - gitMaterialHistoryRepositoryImpl := repository6.NewGitMaterialHistoryRepositoyImpl(db) + gitMaterialHistoryRepositoryImpl := repository8.NewGitMaterialHistoryRepositoyImpl(db) gitMaterialHistoryServiceImpl := history.NewGitMaterialHistoryServiceImpl(gitMaterialHistoryRepositoryImpl, sugaredLogger) - ciPipelineHistoryRepositoryImpl := repository6.NewCiPipelineHistoryRepositoryImpl(db, sugaredLogger) + ciPipelineHistoryRepositoryImpl := repository8.NewCiPipelineHistoryRepositoryImpl(db, sugaredLogger) ciPipelineHistoryServiceImpl := history.NewCiPipelineHistoryServiceImpl(ciPipelineHistoryRepositoryImpl, sugaredLogger, ciPipelineRepositoryImpl) ciBuildConfigRepositoryImpl := pipelineConfig.NewCiBuildConfigRepositoryImpl(db, sugaredLogger) ciBuildConfigServiceImpl := pipeline.NewCiBuildConfigServiceImpl(sugaredLogger, ciBuildConfigRepositoryImpl) ciTemplateServiceImpl := pipeline.NewCiTemplateServiceImpl(sugaredLogger, ciBuildConfigServiceImpl, ciTemplateRepositoryImpl, ciTemplateOverrideRepositoryImpl) + chartRepoRepositoryImpl := chartRepoRepository.NewChartRepoRepositoryImpl(db) + utilMergeUtil := util.MergeUtil{ + Logger: sugaredLogger, + } configMapServiceImpl := pipeline.NewConfigMapServiceImpl(chartRepositoryImpl, sugaredLogger, chartRepoRepositoryImpl, utilMergeUtil, pipelineConfigRepositoryImpl, configMapRepositoryImpl, envConfigOverrideRepositoryImpl, commonServiceImpl, appRepositoryImpl, configMapHistoryServiceImpl, environmentRepositoryImpl, scopedVariableCMCSManagerImpl) ciCdPipelineOrchestratorImpl := pipeline.NewCiCdPipelineOrchestrator(appRepositoryImpl, sugaredLogger, materialRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciCdConfig, appWorkflowRepositoryImpl, environmentRepositoryImpl, attributesServiceImpl, appListingRepositoryImpl, appCrudOperationServiceImpl, userAuthServiceImpl, prePostCdScriptHistoryServiceImpl, prePostCiScriptHistoryServiceImpl, pipelineStageServiceImpl, ciTemplateOverrideRepositoryImpl, gitMaterialHistoryServiceImpl, ciPipelineHistoryServiceImpl, ciTemplateServiceImpl, dockerArtifactStoreRepositoryImpl, ciArtifactRepositoryImpl, configMapServiceImpl, customTagServiceImpl, genericNoteServiceImpl) ecrConfig, err := pipeline.GetEcrConfig() if err != nil { return nil, err } - ciTemplateHistoryRepositoryImpl := repository6.NewCiTemplateHistoryRepositoryImpl(db, sugaredLogger) + ciTemplateHistoryRepositoryImpl := repository8.NewCiTemplateHistoryRepositoryImpl(db, sugaredLogger) ciTemplateHistoryServiceImpl := history.NewCiTemplateHistoryServiceImpl(ciTemplateHistoryRepositoryImpl, sugaredLogger) resourceGroupRepositoryImpl := resourceGroup.NewResourceGroupRepositoryImpl(db) resourceGroupMappingRepositoryImpl := resourceGroup.NewResourceGroupMappingRepositoryImpl(db) @@ -501,19 +481,20 @@ func InitializeApp() (*App, error) { ciMaterialConfigServiceImpl := pipeline.NewCiMaterialConfigServiceImpl(sugaredLogger, materialRepositoryImpl, ciTemplateServiceImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, gitMaterialHistoryServiceImpl, pipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl) imageTaggingRepositoryImpl := repository13.NewImageTaggingRepositoryImpl(db) imageTaggingServiceImpl := pipeline.NewImageTaggingServiceImpl(imageTaggingRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, sugaredLogger) - propertiesConfigServiceImpl := pipeline.NewPropertiesConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, chartRefRepositoryImpl, utilMergeUtil, environmentRepositoryImpl, ciCdPipelineOrchestratorImpl, applicationServiceClientImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl) - pipelineDeploymentServiceTypeConfig, err := pipeline.GetDeploymentServiceTypeConfig() + propertiesConfigServiceImpl := pipeline.NewPropertiesConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl) + deploymentServiceTypeConfig, err := pipeline.GetDeploymentServiceTypeConfig() if err != nil { return nil, err } devtronAppCMCSServiceImpl := pipeline.NewDevtronAppCMCSServiceImpl(sugaredLogger, appServiceImpl, attributesRepositoryImpl) - cdPipelineConfigServiceImpl := pipeline.NewCdPipelineConfigServiceImpl(sugaredLogger, pipelineRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, appRepositoryImpl, appServiceImpl, deploymentGroupRepositoryImpl, ciCdPipelineOrchestratorImpl, appStatusRepositoryImpl, ciPipelineRepositoryImpl, prePostCdScriptHistoryServiceImpl, clusterRepositoryImpl, helmAppServiceImpl, enforcerUtilImpl, gitOpsConfigRepositoryImpl, pipelineStrategyHistoryServiceImpl, chartRepositoryImpl, resourceGroupServiceImpl, chartTemplateServiceImpl, propertiesConfigServiceImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, pipelineDeploymentServiceTypeConfig, applicationServiceClientImpl, customTagServiceImpl, pipelineConfigListenerServiceImpl, devtronAppCMCSServiceImpl, ciPipelineConfigServiceImpl, buildPipelineSwitchServiceImpl, argoClientWrapperServiceImpl, deployedAppMetricsServiceImpl) + cdPipelineConfigServiceImpl := pipeline.NewCdPipelineConfigServiceImpl(sugaredLogger, pipelineRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, appRepositoryImpl, appServiceImpl, deploymentGroupRepositoryImpl, ciCdPipelineOrchestratorImpl, appStatusRepositoryImpl, ciPipelineRepositoryImpl, prePostCdScriptHistoryServiceImpl, clusterRepositoryImpl, helmAppServiceImpl, enforcerUtilImpl, gitOpsConfigRepositoryImpl, pipelineStrategyHistoryServiceImpl, chartRepositoryImpl, resourceGroupServiceImpl, chartTemplateServiceImpl, propertiesConfigServiceImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deploymentServiceTypeConfig, applicationServiceClientImpl, customTagServiceImpl, pipelineConfigListenerServiceImpl, devtronAppCMCSServiceImpl, ciPipelineConfigServiceImpl, buildPipelineSwitchServiceImpl, argoClientWrapperServiceImpl, deployedAppMetricsServiceImpl) appArtifactManagerImpl := pipeline.NewAppArtifactManagerImpl(sugaredLogger, cdWorkflowRepositoryImpl, userServiceImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, ciWorkflowRepositoryImpl, pipelineStageServiceImpl, cdPipelineConfigServiceImpl, dockerArtifactStoreRepositoryImpl, ciPipelineRepositoryImpl, ciTemplateServiceImpl) globalStrategyMetadataChartRefMappingRepositoryImpl := chartRepoRepository.NewGlobalStrategyMetadataChartRefMappingRepositoryImpl(db, sugaredLogger) devtronAppStrategyServiceImpl := pipeline.NewDevtronAppStrategyServiceImpl(sugaredLogger, chartRepositoryImpl, globalStrategyMetadataChartRefMappingRepositoryImpl, ciCdPipelineOrchestratorImpl, cdPipelineConfigServiceImpl) appDeploymentTypeChangeManagerImpl := pipeline.NewAppDeploymentTypeChangeManagerImpl(sugaredLogger, pipelineRepositoryImpl, workflowDagExecutorImpl, appServiceImpl, chartTemplateServiceImpl, appStatusRepositoryImpl, helmAppServiceImpl, applicationServiceClientImpl, appArtifactManagerImpl, cdPipelineConfigServiceImpl) devtronAppConfigServiceImpl := pipeline.NewDevtronAppConfigServiceImpl(sugaredLogger, ciCdPipelineOrchestratorImpl, appRepositoryImpl, pipelineRepositoryImpl, resourceGroupServiceImpl, enforcerUtilImpl, ciMaterialConfigServiceImpl) pipelineBuilderImpl := pipeline.NewPipelineBuilderImpl(sugaredLogger, materialRepositoryImpl, chartRepositoryImpl, ciPipelineConfigServiceImpl, ciMaterialConfigServiceImpl, appArtifactManagerImpl, devtronAppCMCSServiceImpl, devtronAppStrategyServiceImpl, appDeploymentTypeChangeManagerImpl, cdPipelineConfigServiceImpl, devtronAppConfigServiceImpl) + chartServiceImpl := chart.NewChartServiceImpl(chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, appRepositoryImpl, utilMergeUtil, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, environmentRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) dbMigrationServiceImpl := pipeline.NewDbMogrationService(sugaredLogger, dbMigrationConfigRepositoryImpl) ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciWorkflowRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, mergeUtil, ciPipelineRepositoryImpl, prePostCiScriptHistoryServiceImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateServiceImpl, appCrudOperationServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, customTagServiceImpl, pluginInputVariableParserImpl, globalPluginServiceImpl) ciLogServiceImpl, err := pipeline.NewCiLogServiceImpl(sugaredLogger, ciServiceImpl, k8sUtil) @@ -531,11 +512,11 @@ func InitializeApp() (*App, error) { appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, appRepositoryImpl, userAuthServiceImpl) appCloneServiceImpl := appClone.NewAppCloneServiceImpl(sugaredLogger, pipelineBuilderImpl, materialRepositoryImpl, chartServiceImpl, configMapServiceImpl, appWorkflowServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, ciTemplateOverrideRepositoryImpl, pipelineStageServiceImpl, ciTemplateServiceImpl, appRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, appWorkflowRepositoryImpl, ciPipelineConfigServiceImpl) deploymentTemplateRepositoryImpl := repository.NewDeploymentTemplateRepositoryImpl(db, sugaredLogger) - deploymentTemplateServiceImpl := generateManifest.NewDeploymentTemplateServiceImpl(sugaredLogger, chartServiceImpl, appListingServiceImpl, appListingRepositoryImpl, deploymentTemplateRepositoryImpl, helmAppServiceImpl, chartRepositoryImpl, chartTemplateServiceImpl, helmAppClientImpl, k8sUtil, propertiesConfigServiceImpl, deploymentTemplateHistoryServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl) + deploymentTemplateServiceImpl := generateManifest.NewDeploymentTemplateServiceImpl(sugaredLogger, chartServiceImpl, appListingServiceImpl, appListingRepositoryImpl, deploymentTemplateRepositoryImpl, helmAppServiceImpl, chartRepositoryImpl, chartTemplateServiceImpl, helmAppClientImpl, k8sUtil, propertiesConfigServiceImpl, deploymentTemplateHistoryServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, chartRefServiceImpl) imageScanObjectMetaRepositoryImpl := security.NewImageScanObjectMetaRepositoryImpl(db, sugaredLogger) cveStoreRepositoryImpl := security.NewCveStoreRepositoryImpl(db, sugaredLogger) policyServiceImpl := security2.NewPolicyServiceImpl(environmentServiceImpl, sugaredLogger, appRepositoryImpl, pipelineOverrideRepositoryImpl, cvePolicyRepositoryImpl, clusterServiceImplExtended, pipelineRepositoryImpl, imageScanResultRepositoryImpl, imageScanDeployInfoRepositoryImpl, imageScanObjectMetaRepositoryImpl, httpClient, ciArtifactRepositoryImpl, ciCdConfig, imageScanHistoryRepositoryImpl, cveStoreRepositoryImpl, ciTemplateRepositoryImpl) - pipelineConfigRestHandlerImpl := app3.NewPipelineRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, chartServiceImpl, propertiesConfigServiceImpl, dbMigrationServiceImpl, applicationServiceClientImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, environmentServiceImpl, gitRegistryConfigImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, deploymentTemplateServiceImpl, appWorkflowServiceImpl, materialRepositoryImpl, policyServiceImpl, imageScanResultRepositoryImpl, gitProviderRepositoryImpl, argoUserServiceImpl, ciPipelineMaterialRepositoryImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, deployedAppMetricsServiceImpl) + pipelineConfigRestHandlerImpl := app3.NewPipelineRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, chartServiceImpl, propertiesConfigServiceImpl, dbMigrationServiceImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, environmentServiceImpl, gitRegistryConfigImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, deploymentTemplateServiceImpl, appWorkflowServiceImpl, materialRepositoryImpl, policyServiceImpl, imageScanResultRepositoryImpl, gitProviderRepositoryImpl, argoUserServiceImpl, ciPipelineMaterialRepositoryImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) appWorkflowRestHandlerImpl := restHandler.NewAppWorkflowRestHandlerImpl(sugaredLogger, userServiceImpl, appWorkflowServiceImpl, teamServiceImpl, enforcerImpl, pipelineBuilderImpl, appRepositoryImpl, enforcerUtilImpl) webhookEventDataRepositoryImpl := repository.NewWebhookEventDataRepositoryImpl(db) webhookEventDataConfigImpl := pipeline.NewWebhookEventDataConfigImpl(sugaredLogger, webhookEventDataRepositoryImpl) @@ -549,7 +530,23 @@ func InitializeApp() (*App, error) { migrateDbRestHandlerImpl := restHandler.NewMigrateDbRestHandlerImpl(dockerRegistryConfigImpl, sugaredLogger, gitRegistryConfigImpl, dbConfigServiceImpl, userServiceImpl, validate, dbMigrationServiceImpl, enforcerImpl) migrateDbRouterImpl := router.NewMigrateDbRouterImpl(migrateDbRestHandlerImpl) appStoreVersionValuesRepositoryImpl := appStoreValuesRepository.NewAppStoreVersionValuesRepositoryImpl(sugaredLogger, db) - appStoreValuesServiceImpl := service2.NewAppStoreValuesServiceImpl(sugaredLogger, appStoreApplicationVersionRepositoryImpl, installedAppRepositoryImpl, appStoreVersionValuesRepositoryImpl, userServiceImpl) + appStoreValuesServiceImpl := service.NewAppStoreValuesServiceImpl(sugaredLogger, appStoreApplicationVersionRepositoryImpl, installedAppRepositoryImpl, appStoreVersionValuesRepositoryImpl, userServiceImpl) + acdAuthConfig, err := util3.GetACDAuthConfig() + if err != nil { + return nil, err + } + tokenCache := util3.NewTokenCache(sugaredLogger, acdAuthConfig, userAuthServiceImpl) + chartGroupDeploymentRepositoryImpl := repository3.NewChartGroupDeploymentRepositoryImpl(db, sugaredLogger) + appStoreDeploymentCommonServiceImpl := appStoreDeploymentCommon.NewAppStoreDeploymentCommonServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, chartTemplateServiceImpl, gitFactory, gitOpsConfigRepositoryImpl) + appStoreDeploymentFullModeServiceImpl := appStoreDeploymentFullMode.NewAppStoreDeploymentFullModeServiceImpl(sugaredLogger, chartTemplateServiceImpl, repositoryServiceClientImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, applicationServiceClientImpl, argoK8sClientImpl, gitFactory, acdAuthConfig, globalEnvVariables, installedAppRepositoryImpl, tokenCache, argoUserServiceImpl, gitOpsConfigRepositoryImpl, pipelineStatusTimelineServiceImpl, appStoreDeploymentCommonServiceImpl, argoClientWrapperServiceImpl, pubSubClientServiceImpl, installedAppVersionHistoryRepositoryImpl, acdConfig) + clusterInstalledAppsRepositoryImpl := repository3.NewClusterInstalledAppsRepositoryImpl(db, sugaredLogger) + appStoreDeploymentHelmServiceImpl := appStoreDeploymentTool.NewAppStoreDeploymentHelmServiceImpl(sugaredLogger, helmAppServiceImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, helmAppClientImpl, installedAppRepositoryImpl, appStoreDeploymentCommonServiceImpl, ociRegistryConfigRepositoryImpl) + appStoreDeploymentArgoCdServiceImpl := appStoreDeploymentGitopsTool.NewAppStoreDeploymentArgoCdServiceImpl(sugaredLogger, appStoreDeploymentFullModeServiceImpl, applicationServiceClientImpl, chartGroupDeploymentRepositoryImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, chartTemplateServiceImpl, gitFactory, argoUserServiceImpl, appStoreDeploymentCommonServiceImpl, helmAppServiceImpl, gitOpsConfigRepositoryImpl, appStatusServiceImpl, pipelineStatusTimelineServiceImpl, userServiceImpl, pipelineStatusTimelineRepositoryImpl, appStoreApplicationVersionRepositoryImpl, argoClientWrapperServiceImpl, acdConfig) + serviceDeploymentServiceTypeConfig, err := service2.GetDeploymentServiceTypeConfig() + if err != nil { + return nil, err + } + appStoreDeploymentServiceImpl := service2.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, clusterInstalledAppsRepositoryImpl, appRepositoryImpl, appStoreDeploymentHelmServiceImpl, appStoreDeploymentArgoCdServiceImpl, environmentServiceImpl, clusterServiceImplExtended, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, globalEnvVariables, installedAppVersionHistoryRepositoryImpl, gitOpsConfigRepositoryImpl, attributesServiceImpl, serviceDeploymentServiceTypeConfig, chartTemplateServiceImpl, acdConfig) k8sResourceHistoryRepositoryImpl := repository14.NewK8sResourceHistoryRepositoryImpl(db, sugaredLogger) k8sResourceHistoryServiceImpl := kubernetesResourceAuditLogs.Newk8sResourceHistoryServiceImpl(k8sResourceHistoryRepositoryImpl, sugaredLogger, appRepositoryImpl, environmentRepositoryImpl) ephemeralContainersRepositoryImpl := repository2.NewEphemeralContainersRepositoryImpl(db) @@ -559,7 +556,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - installedAppServiceImpl, err := service.NewInstalledAppServiceImpl(sugaredLogger, installedAppRepositoryImpl, chartTemplateServiceImpl, refChartProxyDir, repositoryServiceClientImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, teamRepositoryImpl, appRepositoryImpl, applicationServiceClientImpl, appStoreValuesServiceImpl, pubSubClientServiceImpl, tokenCache, chartGroupDeploymentRepositoryImpl, environmentServiceImpl, argoK8sClientImpl, gitFactory, acdAuthConfig, gitOpsConfigRepositoryImpl, userServiceImpl, appStoreDeploymentFullModeServiceImpl, appStoreDeploymentServiceImpl, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, helmAppClientImpl, helmAppServiceImpl, attributesRepositoryImpl, appStatusServiceImpl, k8sUtil, pipelineStatusTimelineServiceImpl, appStoreDeploymentCommonServiceImpl, appStoreDeploymentArgoCdServiceImpl, k8sCommonServiceImpl, k8sApplicationServiceImpl, acdConfig) + installedAppServiceImpl, err := service2.NewInstalledAppServiceImpl(sugaredLogger, installedAppRepositoryImpl, chartTemplateServiceImpl, repositoryServiceClientImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, teamRepositoryImpl, appRepositoryImpl, applicationServiceClientImpl, appStoreValuesServiceImpl, pubSubClientServiceImpl, tokenCache, chartGroupDeploymentRepositoryImpl, environmentServiceImpl, argoK8sClientImpl, gitFactory, acdAuthConfig, gitOpsConfigRepositoryImpl, userServiceImpl, appStoreDeploymentFullModeServiceImpl, appStoreDeploymentServiceImpl, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, helmAppClientImpl, helmAppServiceImpl, appStatusServiceImpl, k8sUtil, pipelineStatusTimelineServiceImpl, appStoreDeploymentCommonServiceImpl, k8sCommonServiceImpl, k8sApplicationServiceImpl, acdConfig) if err != nil { return nil, err } @@ -646,7 +643,7 @@ func InitializeApp() (*App, error) { roleGroupServiceImpl := user.NewRoleGroupServiceImpl(userAuthRepositoryImpl, sugaredLogger, userRepositoryImpl, roleGroupRepositoryImpl, userCommonServiceImpl) userRestHandlerImpl := user2.NewUserRestHandlerImpl(userServiceImpl, validate, sugaredLogger, enforcerImpl, roleGroupServiceImpl, userCommonServiceImpl) userRouterImpl := user2.NewUserRouterImpl(userRestHandlerImpl) - chartRefRestHandlerImpl := restHandler.NewChartRefRestHandlerImpl(chartServiceImpl, sugaredLogger) + chartRefRestHandlerImpl := restHandler.NewChartRefRestHandlerImpl(sugaredLogger, chartRefServiceImpl, chartServiceImpl) chartRefRouterImpl := router.NewChartRefRouterImpl(chartRefRestHandlerImpl) configMapRestHandlerImpl := restHandler.NewConfigMapRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, chartServiceImpl, userServiceImpl, teamServiceImpl, enforcerImpl, pipelineRepositoryImpl, enforcerUtilImpl, configMapServiceImpl) configMapRouterImpl := router.NewConfigMapRouterImpl(configMapRestHandlerImpl) @@ -662,7 +659,7 @@ func InitializeApp() (*App, error) { appStoreDeploymentRouterImpl := appStoreDeployment.NewAppStoreDeploymentRouterImpl(appStoreDeploymentRestHandlerImpl) appStoreStatusTimelineRestHandlerImpl := appStore.NewAppStoreStatusTimelineRestHandlerImpl(sugaredLogger, pipelineStatusTimelineServiceImpl, enforcerUtilImpl, enforcerImpl) appStoreRouterImpl := appStore.NewAppStoreRouterImpl(installedAppRestHandlerImpl, appStoreValuesRouterImpl, appStoreDiscoverRouterImpl, chartProviderRouterImpl, appStoreDeploymentRouterImpl, appStoreStatusTimelineRestHandlerImpl) - chartRepositoryRestHandlerImpl := chartRepo2.NewChartRepositoryRestHandlerImpl(sugaredLogger, userServiceImpl, chartRepositoryServiceImpl, enforcerImpl, validate, deleteServiceExtendedImpl, chartRefRepositoryImpl, refChartDir, attributesServiceImpl) + chartRepositoryRestHandlerImpl := chartRepo2.NewChartRepositoryRestHandlerImpl(sugaredLogger, userServiceImpl, chartRepositoryServiceImpl, enforcerImpl, validate, deleteServiceExtendedImpl, attributesServiceImpl) chartRepositoryRouterImpl := chartRepo2.NewChartRepositoryRouterImpl(chartRepositoryRestHandlerImpl) lensConfig, err := lens.GetLensConfig() if err != nil { @@ -686,7 +683,7 @@ func InitializeApp() (*App, error) { batchOperationRouterImpl := router.NewBatchOperationRouterImpl(batchOperationRestHandlerImpl, sugaredLogger) chartGroupEntriesRepositoryImpl := repository3.NewChartGroupEntriesRepositoryImpl(db, sugaredLogger) chartGroupReposotoryImpl := repository3.NewChartGroupReposotoryImpl(db, sugaredLogger) - chartGroupServiceImpl := service.NewChartGroupServiceImpl(chartGroupEntriesRepositoryImpl, chartGroupReposotoryImpl, sugaredLogger, chartGroupDeploymentRepositoryImpl, installedAppRepositoryImpl, appStoreVersionValuesRepositoryImpl, userAuthServiceImpl) + chartGroupServiceImpl := service2.NewChartGroupServiceImpl(chartGroupEntriesRepositoryImpl, chartGroupReposotoryImpl, sugaredLogger, chartGroupDeploymentRepositoryImpl, installedAppRepositoryImpl, appStoreVersionValuesRepositoryImpl, userAuthServiceImpl) chartGroupRestHandlerImpl := restHandler.NewChartGroupRestHandlerImpl(chartGroupServiceImpl, sugaredLogger, userServiceImpl, enforcerImpl, validate) chartGroupRouterImpl := router.NewChartGroupRouterImpl(chartGroupRestHandlerImpl) testSuitRestHandlerImpl := restHandler.NewTestSuitRestHandlerImpl(sugaredLogger, userServiceImpl, validate, enforcerImpl, enforcerUtilImpl, eventClientConfig, httpClient) @@ -733,7 +730,7 @@ func InitializeApp() (*App, error) { telemetryRestHandlerImpl := restHandler.NewTelemetryRestHandlerImpl(sugaredLogger, telemetryEventClientImplExtended, enforcerImpl, userServiceImpl) telemetryRouterImpl := router.NewTelemetryRouterImpl(sugaredLogger, telemetryRestHandlerImpl) bulkUpdateRepositoryImpl := bulkUpdate.NewBulkUpdateRepository(db, sugaredLogger) - bulkUpdateServiceImpl, err := bulkAction.NewBulkUpdateServiceImpl(bulkUpdateRepositoryImpl, chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, defaultChart, utilMergeUtil, repositoryServiceClientImpl, chartRefRepositoryImpl, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, httpClient, appRepositoryImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, workflowDagExecutorImpl, cdWorkflowRepositoryImpl, pipelineBuilderImpl, helmAppServiceImpl, enforcerUtilImpl, enforcerUtilHelmImpl, ciHandlerImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, appWorkflowServiceImpl, pubSubClientServiceImpl, argoUserServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl) + bulkUpdateServiceImpl, err := bulkAction.NewBulkUpdateServiceImpl(bulkUpdateRepositoryImpl, chartRepositoryImpl, sugaredLogger, environmentRepositoryImpl, pipelineRepositoryImpl, appRepositoryImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, workflowDagExecutorImpl, pipelineBuilderImpl, enforcerUtilImpl, ciHandlerImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, appWorkflowServiceImpl, pubSubClientServiceImpl, argoUserServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) if err != nil { return nil, err } @@ -752,7 +749,7 @@ func InitializeApp() (*App, error) { k8sApplicationRouterImpl := application3.NewK8sApplicationRouterImpl(k8sApplicationRestHandlerImpl) pProfRestHandlerImpl := restHandler.NewPProfRestHandler(userServiceImpl, enforcerImpl) pProfRouterImpl := router.NewPProfRouter(sugaredLogger, pProfRestHandlerImpl) - deploymentConfigRestHandlerImpl := deployment.NewDeploymentConfigRestHandlerImpl(sugaredLogger, userServiceImpl, enforcerImpl, validate, refChartDir, chartServiceImpl, chartRefRepositoryImpl) + deploymentConfigRestHandlerImpl := deployment.NewDeploymentConfigRestHandlerImpl(sugaredLogger, userServiceImpl, enforcerImpl, chartServiceImpl, chartRefServiceImpl) deploymentConfigRouterImpl := deployment.NewDeploymentRouterImpl(deploymentConfigRestHandlerImpl) dashboardTelemetryRestHandlerImpl := dashboardEvent.NewDashboardTelemetryRestHandlerImpl(sugaredLogger, telemetryEventClientImplExtended) dashboardTelemetryRouterImpl := dashboardEvent.NewDashboardTelemetryRouterImpl(dashboardTelemetryRestHandlerImpl) @@ -826,10 +823,3 @@ func InitializeApp() (*App, error) { mainApp := NewApp(muxRouter, sugaredLogger, sseSSE, syncedEnforcer, db, pubSubClientServiceImpl, sessionManager, posthogClient, loggingMiddlewareImpl) return mainApp, nil } - -var ( - _wireChartWorkingDirValue = util.ChartWorkingDir("/tmp/charts/") - _wireRefChartDirValue = chartRepoRepository.RefChartDir("scripts/devtron-reference-helm-charts") - _wireDefaultChartValue = chart.DefaultChart("reference-app-rolling") - _wireRefChartProxyDirValue = appStoreBean.RefChartProxyDir("scripts/devtron-reference-helm-charts") -) From a1421a57260133ce60113aec356ca17bf64c9642 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Mon, 15 Jan 2024 18:28:02 +0530 Subject: [PATCH 07/64] removed infra metrics db calls --- .../repository/AppLevelMetricsRepository.go | 10 ++-- .../sql/repository/AppListingRepository.go | 2 + .../EnvLevelAppMetricsRepository.go | 12 ++--- pkg/app/AppListingService.go | 53 ++++++------------- .../deployedAppMetrics/DeployedAppMetrics.go | 13 ++--- wire_gen.go | 2 +- 6 files changed, 36 insertions(+), 56 deletions(-) diff --git a/internal/sql/repository/AppLevelMetricsRepository.go b/internal/sql/repository/AppLevelMetricsRepository.go index 6e3568e11a..1a67deefdd 100644 --- a/internal/sql/repository/AppLevelMetricsRepository.go +++ b/internal/sql/repository/AppLevelMetricsRepository.go @@ -24,11 +24,11 @@ import ( ) type AppLevelMetrics struct { - tableName struct{} `sql:"app_level_metrics" pg:",discard_unknown_columns"` - Id int `sql:"id,pk"` - AppId int `sql:"app_id,notnull"` - AppMetrics bool `sql:"app_metrics,notnull"` - InfraMetrics bool `sql:"infra_metrics,notnull"` + tableName struct{} `sql:"app_level_metrics" pg:",discard_unknown_columns"` + Id int `sql:"id,pk"` + AppId int `sql:"app_id,notnull"` + AppMetrics bool `sql:"app_metrics,notnull"` + //InfraMetrics bool `sql:"infra_metrics,notnull"` not being used sql.AuditLog } diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index 2b4a9f84f6..c5e2583b53 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -663,6 +663,7 @@ func (impl AppListingRepositoryImpl) makeAppStageStatus(stage int, stageName str func (impl AppListingRepositoryImpl) FetchOtherEnvironment(appId int) ([]*bean.Environment, error) { // other environment tab var otherEnvironments []*bean.Environment + //TODO: remove infra metrics from query as it is not being used from here query := `select pcwr.pipeline_id, pcwr.last_deployed, pcwr.latest_cd_workflow_runner_id, pcwr.environment_id, pcwr.deployment_app_delete_request, e.cluster_id, e.environment_name, e.default as prod, e.description, ca.image as last_deployed_image, u.email_id as last_deployed_by, elam.app_metrics, elam.infra_metrics, ap.status as app_status @@ -691,6 +692,7 @@ func (impl AppListingRepositoryImpl) FetchOtherEnvironment(appId int) ([]*bean.E func (impl AppListingRepositoryImpl) FetchMinDetailOtherEnvironment(appId int) ([]*bean.Environment, error) { impl.Logger.Debug("reached at FetchMinDetailOtherEnvironment:") var otherEnvironments []*bean.Environment + //TODO: remove infra metrics from query as it is not being used from here query := `SELECT p.environment_id,env.environment_name,env.description,env.is_virtual_environment, env.cluster_id, env.default as prod, p.deployment_app_delete_request, env_app_m.app_metrics,env_app_m.infra_metrics from (SELECT pl.id,pl.app_id,pl.environment_id,pl.deleted, pl.deployment_app_delete_request from pipeline pl diff --git a/internal/sql/repository/EnvLevelAppMetricsRepository.go b/internal/sql/repository/EnvLevelAppMetricsRepository.go index f849212fab..bb81432024 100644 --- a/internal/sql/repository/EnvLevelAppMetricsRepository.go +++ b/internal/sql/repository/EnvLevelAppMetricsRepository.go @@ -24,12 +24,12 @@ import ( ) type EnvLevelAppMetrics struct { - tableName struct{} `sql:"env_level_app_metrics" pg:",discard_unknown_columns"` - Id int `sql:"id,pk"` - AppId int `sql:"app_id,notnull"` - EnvId int `sql:"env_id,notnull"` - AppMetrics *bool `sql:"app_metrics,notnull"` - InfraMetrics *bool `sql:"infra_metrics,notnull"` + tableName struct{} `sql:"env_level_app_metrics" pg:",discard_unknown_columns"` + Id int `sql:"id,pk"` + AppId int `sql:"app_id,notnull"` + EnvId int `sql:"env_id,notnull"` + AppMetrics *bool `sql:"app_metrics,notnull"` + //InfraMetrics *bool `sql:"infra_metrics,notnull"` not being used sql.AuditLog } diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index 207bec1fd3..462e851cef 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -21,6 +21,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" "net/http" "strconv" "strings" @@ -159,7 +160,6 @@ type AppListingServiceImpl struct { pipelineRepository pipelineConfig.PipelineRepository cdWorkflowRepository pipelineConfig.CdWorkflowRepository linkoutsRepository repository.LinkoutsRepository - appLevelMetricsRepository repository.AppLevelMetricsRepository pipelineOverrideRepository chartConfig.PipelineOverrideRepository environmentRepository repository2.EnvironmentRepository argoUserService argo.ArgoUserService @@ -168,17 +168,18 @@ type AppListingServiceImpl struct { ciPipelineRepository pipelineConfig.CiPipelineRepository dockerRegistryIpsConfigService dockerRegistry.DockerRegistryIpsConfigService userRepository userrepository.UserRepository + deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService } func NewAppListingServiceImpl(Logger *zap.SugaredLogger, appListingRepository repository.AppListingRepository, application application2.ServiceClient, appRepository app.AppRepository, appListingViewBuilder AppListingViewBuilder, pipelineRepository pipelineConfig.PipelineRepository, - linkoutsRepository repository.LinkoutsRepository, appLevelMetricsRepository repository.AppLevelMetricsRepository, - cdWorkflowRepository pipelineConfig.CdWorkflowRepository, + linkoutsRepository repository.LinkoutsRepository, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, pipelineOverrideRepository chartConfig.PipelineOverrideRepository, environmentRepository repository2.EnvironmentRepository, argoUserService argo.ArgoUserService, envOverrideRepository chartConfig.EnvConfigOverrideRepository, chartRepository chartRepoRepository.ChartRepository, ciPipelineRepository pipelineConfig.CiPipelineRepository, - dockerRegistryIpsConfigService dockerRegistry.DockerRegistryIpsConfigService, userRepository userrepository.UserRepository) *AppListingServiceImpl { + dockerRegistryIpsConfigService dockerRegistry.DockerRegistryIpsConfigService, userRepository userrepository.UserRepository, + deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService) *AppListingServiceImpl { serviceImpl := &AppListingServiceImpl{ Logger: Logger, appListingRepository: appListingRepository, @@ -187,7 +188,6 @@ func NewAppListingServiceImpl(Logger *zap.SugaredLogger, appListingRepository re appListingViewBuilder: appListingViewBuilder, pipelineRepository: pipelineRepository, linkoutsRepository: linkoutsRepository, - appLevelMetricsRepository: appLevelMetricsRepository, cdWorkflowRepository: cdWorkflowRepository, pipelineOverrideRepository: pipelineOverrideRepository, environmentRepository: environmentRepository, @@ -197,6 +197,7 @@ func NewAppListingServiceImpl(Logger *zap.SugaredLogger, appListingRepository re ciPipelineRepository: ciPipelineRepository, dockerRegistryIpsConfigService: dockerRegistryIpsConfigService, userRepository: userRepository, + deployedAppMetricsService: deployedAppMetricsService, } return serviceImpl } @@ -1670,21 +1671,13 @@ func (impl AppListingServiceImpl) FetchOtherEnvironment(ctx context.Context, app impl.Logger.Errorw("err", err) return envs, err } - appLevelAppMetrics := false //default value - appLevelInfraMetrics := true //default val + appLevelInfraMetrics := true //default val, not being derived from DB. TODO: remove this from FE since this is derived from prometheus config at cluster level and this logic is already present at FE newCtx, span = otel.Tracer("appLevelMetricsRepository").Start(newCtx, "FindByAppId") - appLevelMetrics, err := impl.appLevelMetricsRepository.FindByAppId(appId) + appLevelAppMetrics, err := impl.deployedAppMetricsService.GetMetricsFlagByAppIdEvenIfNotInDb(appId) span.End() - if err != nil && !util.IsErrNoRows(err) { - impl.Logger.Errorw("error in fetching app metrics", "err", err) + if err != nil { + impl.Logger.Errorw("error, GetMetricsFlagByAppIdEvenIfNotInDb", "err", err, "appId", appId) return envs, err - } else if util.IsErrNoRows(err) { - //populate default val - appLevelAppMetrics = false //default value - appLevelInfraMetrics = true //default val - } else { - appLevelAppMetrics = appLevelMetrics.AppMetrics - appLevelInfraMetrics = appLevelMetrics.InfraMetrics } newCtx, span = otel.Tracer("chartRepository").Start(newCtx, "FindLatestChartForAppByAppId") chart, err := impl.chartRepository.FindLatestChartForAppByAppId(appId) @@ -1709,9 +1702,7 @@ func (impl AppListingServiceImpl) FetchOtherEnvironment(ctx context.Context, app if env.AppMetrics == nil { env.AppMetrics = &appLevelAppMetrics } - if env.InfraMetrics == nil { - env.InfraMetrics = &appLevelInfraMetrics - } + env.InfraMetrics = &appLevelInfraMetrics //using default value, discarding value got from query } return envs, nil } @@ -1722,19 +1713,11 @@ func (impl AppListingServiceImpl) FetchMinDetailOtherEnvironment(appId int) ([]* impl.Logger.Errorw("err", err) return envs, err } - appLevelAppMetrics := false //default value - appLevelInfraMetrics := true //default val - appLevelMetrics, err := impl.appLevelMetricsRepository.FindByAppId(appId) - if err != nil && !util.IsErrNoRows(err) { - impl.Logger.Errorw("error in fetching app metrics", "err", err) - return envs, err - } else if util.IsErrNoRows(err) { - //populate default val - appLevelAppMetrics = false //default value - appLevelInfraMetrics = true //default val - } else { - appLevelAppMetrics = appLevelMetrics.AppMetrics - appLevelInfraMetrics = appLevelMetrics.InfraMetrics + appLevelInfraMetrics := true //default val, not being derived from DB. TODO: remove this from FE since this is derived from prometheus config at cluster level and this logic is already present at FE + appLevelAppMetrics, err := impl.deployedAppMetricsService.GetMetricsFlagByAppIdEvenIfNotInDb(appId) + if err != nil { + impl.Logger.Errorw("error, GetMetricsFlagByAppIdEvenIfNotInDb", "err", err, "appId", appId) + return nil, err } chartRefId, err := impl.chartRepository.FindChartRefIdForLatestChartForAppByAppId(appId) @@ -1764,9 +1747,7 @@ func (impl AppListingServiceImpl) FetchMinDetailOtherEnvironment(appId int) ([]* if env.AppMetrics == nil { env.AppMetrics = &appLevelAppMetrics } - if env.InfraMetrics == nil { - env.InfraMetrics = &appLevelInfraMetrics - } + env.InfraMetrics = &appLevelInfraMetrics //using default value, discarding value got from query } return envs, nil } diff --git a/pkg/deployment/manifest/deployedAppMetrics/DeployedAppMetrics.go b/pkg/deployment/manifest/deployedAppMetrics/DeployedAppMetrics.go index b04de22cff..1f2d38e52b 100644 --- a/pkg/deployment/manifest/deployedAppMetrics/DeployedAppMetrics.go +++ b/pkg/deployment/manifest/deployedAppMetrics/DeployedAppMetrics.go @@ -160,9 +160,8 @@ func (impl *DeployedAppMetricsServiceImpl) createOrUpdateAppLevelMetrics(req *be return existingAppLevelMetrics, nil } else { appLevelMetricsNew := &interalRepo.AppLevelMetrics{ - AppId: req.AppId, - AppMetrics: req.EnableMetrics, - InfraMetrics: true, + AppId: req.AppId, + AppMetrics: req.EnableMetrics, AuditLog: sql.AuditLog{ CreatedOn: time.Now(), UpdatedOn: time.Now(), @@ -187,12 +186,10 @@ func (impl *DeployedAppMetricsServiceImpl) createOrUpdateEnvLevelMetrics(req *be return nil, err } if envLevelAppMetrics == nil || envLevelAppMetrics.Id == 0 { - infraMetrics := true envLevelAppMetrics = &interalRepo.EnvLevelAppMetrics{ - AppId: req.AppId, - EnvId: req.EnvId, - AppMetrics: &req.EnableMetrics, - InfraMetrics: &infraMetrics, + AppId: req.AppId, + EnvId: req.EnvId, + AppMetrics: &req.EnableMetrics, AuditLog: sql.AuditLog{ CreatedOn: time.Now(), UpdatedOn: time.Now(), diff --git a/wire_gen.go b/wire_gen.go index d330b3674e..ab2ddd83ba 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -525,7 +525,7 @@ func InitializeApp() (*App, error) { gitRegistryConfigImpl := pipeline.NewGitRegistryConfigImpl(sugaredLogger, gitProviderRepositoryImpl, clientImpl) appListingViewBuilderImpl := app2.NewAppListingViewBuilderImpl(sugaredLogger) linkoutsRepositoryImpl := repository.NewLinkoutsRepositoryImpl(sugaredLogger, db) - appListingServiceImpl := app2.NewAppListingServiceImpl(sugaredLogger, appListingRepositoryImpl, applicationServiceClientImpl, appRepositoryImpl, appListingViewBuilderImpl, pipelineRepositoryImpl, linkoutsRepositoryImpl, appLevelMetricsRepositoryImpl, cdWorkflowRepositoryImpl, pipelineOverrideRepositoryImpl, environmentRepositoryImpl, argoUserServiceImpl, envConfigOverrideRepositoryImpl, chartRepositoryImpl, ciPipelineRepositoryImpl, dockerRegistryIpsConfigServiceImpl, userRepositoryImpl) + appListingServiceImpl := app2.NewAppListingServiceImpl(sugaredLogger, appListingRepositoryImpl, applicationServiceClientImpl, appRepositoryImpl, appListingViewBuilderImpl, pipelineRepositoryImpl, linkoutsRepositoryImpl, cdWorkflowRepositoryImpl, pipelineOverrideRepositoryImpl, environmentRepositoryImpl, argoUserServiceImpl, envConfigOverrideRepositoryImpl, chartRepositoryImpl, ciPipelineRepositoryImpl, dockerRegistryIpsConfigServiceImpl, userRepositoryImpl, deployedAppMetricsServiceImpl) deploymentEventHandlerImpl := app2.NewDeploymentEventHandlerImpl(sugaredLogger, appListingServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl) cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, helmAppServiceImpl, pipelineOverrideRepositoryImpl, workflowDagExecutorImpl, appListingServiceImpl, appListingRepositoryImpl, pipelineStatusTimelineRepositoryImpl, applicationServiceClientImpl, argoUserServiceImpl, deploymentEventHandlerImpl, eventRESTClientImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceImpl, appStatusServiceImpl, enforcerUtilImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, appRepositoryImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sUtil, workflowServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl, customTagServiceImpl, argoClientWrapperServiceImpl, appServiceConfig, acdConfig) appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, appRepositoryImpl, userAuthServiceImpl) From a6123c5c4bbc7787fd019594459a28651382cd86 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Mon, 15 Jan 2024 18:30:24 +0530 Subject: [PATCH 08/64] moved app metrics repositories from /internal to /pkg --- Wire.go | 9 +++++---- .../mocks/AppLevelMetricsRepository.go | 2 +- .../mocks/EnvLevelAppMetricsRepository.go | 2 +- .../AppServiceDeployment_test.go | 10 +++++----- .../deployedAppMetrics/DeployedAppMetrics.go | 18 +++++++++--------- .../repository/AppLevelMetricsRepository.go | 0 .../repository/EnvLevelAppMetricsRepository.go | 0 wire_gen.go | 5 +++-- 8 files changed, 24 insertions(+), 22 deletions(-) rename {internal/sql => pkg/deployment/manifest/deployedAppMetrics}/repository/AppLevelMetricsRepository.go (100%) rename {internal/sql => pkg/deployment/manifest/deployedAppMetrics}/repository/EnvLevelAppMetricsRepository.go (100%) diff --git a/Wire.go b/Wire.go index b4ef34b994..5913a6ab95 100644 --- a/Wire.go +++ b/Wire.go @@ -98,6 +98,7 @@ import ( "github.com/devtron-labs/devtron/pkg/commonService" delete2 "github.com/devtron-labs/devtron/pkg/delete" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" + repository11 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/repository" "github.com/devtron-labs/devtron/pkg/deploymentGroup" "github.com/devtron-labs/devtron/pkg/devtronResource" repository9 "github.com/devtron-labs/devtron/pkg/devtronResource/repository" @@ -557,11 +558,11 @@ func InitializeApp() (*App, error) { restHandler.NewExternalCiRestHandlerImpl, wire.Bind(new(restHandler.ExternalCiRestHandler), new(*restHandler.ExternalCiRestHandlerImpl)), - repository.NewAppLevelMetricsRepositoryImpl, - wire.Bind(new(repository.AppLevelMetricsRepository), new(*repository.AppLevelMetricsRepositoryImpl)), + repository11.NewAppLevelMetricsRepositoryImpl, + wire.Bind(new(repository11.AppLevelMetricsRepository), new(*repository11.AppLevelMetricsRepositoryImpl)), - repository.NewEnvLevelAppMetricsRepositoryImpl, - wire.Bind(new(repository.EnvLevelAppMetricsRepository), new(*repository.EnvLevelAppMetricsRepositoryImpl)), + repository11.NewEnvLevelAppMetricsRepositoryImpl, + wire.Bind(new(repository11.EnvLevelAppMetricsRepository), new(*repository11.EnvLevelAppMetricsRepositoryImpl)), grafana.GetGrafanaClientConfig, grafana.NewGrafanaClientImpl, diff --git a/internal/sql/repository/mocks/AppLevelMetricsRepository.go b/internal/sql/repository/mocks/AppLevelMetricsRepository.go index 4254e30765..8e1a77c8e1 100644 --- a/internal/sql/repository/mocks/AppLevelMetricsRepository.go +++ b/internal/sql/repository/mocks/AppLevelMetricsRepository.go @@ -3,7 +3,7 @@ package mocks import ( - repository "github.com/devtron-labs/devtron/internal/sql/repository" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/repository" mock "github.com/stretchr/testify/mock" ) diff --git a/internal/sql/repository/mocks/EnvLevelAppMetricsRepository.go b/internal/sql/repository/mocks/EnvLevelAppMetricsRepository.go index 1037a5fd94..6561e22c68 100644 --- a/internal/sql/repository/mocks/EnvLevelAppMetricsRepository.go +++ b/internal/sql/repository/mocks/EnvLevelAppMetricsRepository.go @@ -3,7 +3,7 @@ package mocks import ( - repository "github.com/devtron-labs/devtron/internal/sql/repository" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/repository" mock "github.com/stretchr/testify/mock" ) diff --git a/pkg/app/integrationTest/AppServiceDeployment_test.go b/pkg/app/integrationTest/AppServiceDeployment_test.go index b849021a99..a05fb7ce07 100644 --- a/pkg/app/integrationTest/AppServiceDeployment_test.go +++ b/pkg/app/integrationTest/AppServiceDeployment_test.go @@ -4,7 +4,6 @@ import ( "context" "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/sql/models" - repository3 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" mocks3 "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig/mocks" mocks5 "github.com/devtron-labs/devtron/internal/sql/repository/mocks" @@ -14,6 +13,7 @@ import ( mocks2 "github.com/devtron-labs/devtron/pkg/chartRepo/repository/mocks" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" mocks4 "github.com/devtron-labs/devtron/pkg/cluster/repository/mocks" + repository4 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/repository" "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" "github.com/devtron-labs/devtron/pkg/pipeline/history/repository/mocks" "github.com/devtron-labs/devtron/pkg/sql" @@ -438,7 +438,7 @@ func TestDeploymentTemplateHistoryService(t *testing.T) { mockedAppLevelMetricsRepository := mocks5.NewAppLevelMetricsRepository(t) - appLevelMetricsDBObject := &repository3.AppLevelMetrics{ + appLevelMetricsDBObject := &repository4.AppLevelMetrics{ Id: 1, AppId: 1, AppMetrics: false, @@ -452,7 +452,7 @@ func TestDeploymentTemplateHistoryService(t *testing.T) { appMetrics := true - mockedEnvLevelMetricsDBObject := &repository3.EnvLevelAppMetrics{ + mockedEnvLevelMetricsDBObject := &repository4.EnvLevelAppMetrics{ Id: 1, AppId: 1, EnvId: 1, @@ -521,7 +521,7 @@ func TestDeploymentTemplateHistoryService(t *testing.T) { mockedAppLevelMetricsRepository := mocks5.NewAppLevelMetricsRepository(t) - appLevelMetricsDBObject := &repository3.AppLevelMetrics{ + appLevelMetricsDBObject := &repository4.AppLevelMetrics{ Id: 1, AppId: 1, AppMetrics: false, @@ -533,7 +533,7 @@ func TestDeploymentTemplateHistoryService(t *testing.T) { mockedEnvLevelMetricsRepository := mocks5.NewEnvLevelAppMetricsRepository(t) - mockedEnvLevelMetricsDBObject := &repository3.EnvLevelAppMetrics{ + mockedEnvLevelMetricsDBObject := &repository4.EnvLevelAppMetrics{ Id: 0, AppId: 0, EnvId: 0, diff --git a/pkg/deployment/manifest/deployedAppMetrics/DeployedAppMetrics.go b/pkg/deployment/manifest/deployedAppMetrics/DeployedAppMetrics.go index 1f2d38e52b..b57eb6dafe 100644 --- a/pkg/deployment/manifest/deployedAppMetrics/DeployedAppMetrics.go +++ b/pkg/deployment/manifest/deployedAppMetrics/DeployedAppMetrics.go @@ -2,10 +2,10 @@ package deployedAppMetrics import ( "context" - interalRepo "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/util" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/bean" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/repository" "github.com/devtron-labs/devtron/pkg/sql" "github.com/go-pg/pg" "go.opentelemetry.io/otel" @@ -24,14 +24,14 @@ type DeployedAppMetricsService interface { type DeployedAppMetricsServiceImpl struct { logger *zap.SugaredLogger chartRefRepository chartRepoRepository.ChartRefRepository - appLevelMetricsRepository interalRepo.AppLevelMetricsRepository - envLevelMetricsRepository interalRepo.EnvLevelAppMetricsRepository + appLevelMetricsRepository repository.AppLevelMetricsRepository + envLevelMetricsRepository repository.EnvLevelAppMetricsRepository } func NewDeployedAppMetricsServiceImpl(logger *zap.SugaredLogger, chartRefRepository chartRepoRepository.ChartRefRepository, - appLevelMetricsRepository interalRepo.AppLevelMetricsRepository, - envLevelMetricsRepository interalRepo.EnvLevelAppMetricsRepository) *DeployedAppMetricsServiceImpl { + appLevelMetricsRepository repository.AppLevelMetricsRepository, + envLevelMetricsRepository repository.EnvLevelAppMetricsRepository) *DeployedAppMetricsServiceImpl { return &DeployedAppMetricsServiceImpl{ logger: logger, chartRefRepository: chartRefRepository, @@ -142,7 +142,7 @@ func (impl *DeployedAppMetricsServiceImpl) checkIsAppMetricsSupported(chartRefId return chartRefValue.IsAppMetricsSupported, nil } -func (impl *DeployedAppMetricsServiceImpl) createOrUpdateAppLevelMetrics(req *bean.DeployedAppMetricsRequest) (*interalRepo.AppLevelMetrics, error) { +func (impl *DeployedAppMetricsServiceImpl) createOrUpdateAppLevelMetrics(req *bean.DeployedAppMetricsRequest) (*repository.AppLevelMetrics, error) { existingAppLevelMetrics, err := impl.appLevelMetricsRepository.FindByAppId(req.AppId) if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in app metrics app level flag", "error", err) @@ -159,7 +159,7 @@ func (impl *DeployedAppMetricsServiceImpl) createOrUpdateAppLevelMetrics(req *be } return existingAppLevelMetrics, nil } else { - appLevelMetricsNew := &interalRepo.AppLevelMetrics{ + appLevelMetricsNew := &repository.AppLevelMetrics{ AppId: req.AppId, AppMetrics: req.EnableMetrics, AuditLog: sql.AuditLog{ @@ -178,7 +178,7 @@ func (impl *DeployedAppMetricsServiceImpl) createOrUpdateAppLevelMetrics(req *be } } -func (impl *DeployedAppMetricsServiceImpl) createOrUpdateEnvLevelMetrics(req *bean.DeployedAppMetricsRequest) (*interalRepo.EnvLevelAppMetrics, error) { +func (impl *DeployedAppMetricsServiceImpl) createOrUpdateEnvLevelMetrics(req *bean.DeployedAppMetricsRequest) (*repository.EnvLevelAppMetrics, error) { // update and create env level app metrics envLevelAppMetrics, err := impl.envLevelMetricsRepository.FindByAppIdAndEnvId(req.AppId, req.EnvId) if err != nil && err != pg.ErrNoRows { @@ -186,7 +186,7 @@ func (impl *DeployedAppMetricsServiceImpl) createOrUpdateEnvLevelMetrics(req *be return nil, err } if envLevelAppMetrics == nil || envLevelAppMetrics.Id == 0 { - envLevelAppMetrics = &interalRepo.EnvLevelAppMetrics{ + envLevelAppMetrics = &repository.EnvLevelAppMetrics{ AppId: req.AppId, EnvId: req.EnvId, AppMetrics: &req.EnableMetrics, diff --git a/internal/sql/repository/AppLevelMetricsRepository.go b/pkg/deployment/manifest/deployedAppMetrics/repository/AppLevelMetricsRepository.go similarity index 100% rename from internal/sql/repository/AppLevelMetricsRepository.go rename to pkg/deployment/manifest/deployedAppMetrics/repository/AppLevelMetricsRepository.go diff --git a/internal/sql/repository/EnvLevelAppMetricsRepository.go b/pkg/deployment/manifest/deployedAppMetrics/repository/EnvLevelAppMetricsRepository.go similarity index 100% rename from internal/sql/repository/EnvLevelAppMetricsRepository.go rename to pkg/deployment/manifest/deployedAppMetrics/repository/EnvLevelAppMetricsRepository.go diff --git a/wire_gen.go b/wire_gen.go index ab2ddd83ba..57c5566906 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -103,6 +103,7 @@ import ( "github.com/devtron-labs/devtron/pkg/commonService" delete2 "github.com/devtron-labs/devtron/pkg/delete" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" + repository15 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/repository" "github.com/devtron-labs/devtron/pkg/deploymentGroup" "github.com/devtron-labs/devtron/pkg/devtronResource" repository8 "github.com/devtron-labs/devtron/pkg/devtronResource/repository" @@ -358,8 +359,8 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - appLevelMetricsRepositoryImpl := repository.NewAppLevelMetricsRepositoryImpl(db, sugaredLogger) - envLevelAppMetricsRepositoryImpl := repository.NewEnvLevelAppMetricsRepositoryImpl(db, sugaredLogger) + appLevelMetricsRepositoryImpl := repository15.NewAppLevelMetricsRepositoryImpl(db, sugaredLogger) + envLevelAppMetricsRepositoryImpl := repository15.NewEnvLevelAppMetricsRepositoryImpl(db, sugaredLogger) deployedAppMetricsServiceImpl := deployedAppMetrics.NewDeployedAppMetricsServiceImpl(sugaredLogger, chartRefRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl) deploymentTemplateHistoryServiceImpl := history.NewDeploymentTemplateHistoryServiceImpl(sugaredLogger, deploymentTemplateHistoryRepositoryImpl, pipelineRepositoryImpl, chartRepositoryImpl, chartRefRepositoryImpl, userServiceImpl, cdWorkflowRepositoryImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl) chartWorkingDir := _wireChartWorkingDirValue From acfec88e50ac70b14d59c24c4d3d53326a710d55 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Mon, 15 Jan 2024 18:46:20 +0530 Subject: [PATCH 09/64] moved: const and types to bean --- pkg/appStore/bean/bean.go | 17 +++++++ .../deployment/service/InstalledAppService.go | 44 +++++++------------ 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/pkg/appStore/bean/bean.go b/pkg/appStore/bean/bean.go index 373cc85deb..6254551ce7 100644 --- a/pkg/appStore/bean/bean.go +++ b/pkg/appStore/bean/bean.go @@ -359,3 +359,20 @@ type HelmReleaseStatusConfig struct { IsReleaseInstalled bool ErrorInInstallation bool } + +type ChartComponents struct { + ChartComponent []*ChartComponent `json:"charts"` +} + +type ChartComponent struct { + Name string `json:"name"` + Values string `json:"values"` +} + +const ( + DEFAULT_ENVIRONMENT_OR_NAMESPACE_OR_PROJECT = "devtron" + CLUSTER_COMPONENT_DIR_PATH = "/cluster/component" + HELM_RELEASE_STATUS_FAILED = "Failed" + HELM_RELEASE_STATUS_PROGRESSING = "Progressing" + HELM_RELEASE_STATUS_UNKNOWN = "Unknown" +) diff --git a/pkg/appStore/deployment/service/InstalledAppService.go b/pkg/appStore/deployment/service/InstalledAppService.go index cc426a9381..aebe347bcd 100644 --- a/pkg/appStore/deployment/service/InstalledAppService.go +++ b/pkg/appStore/deployment/service/InstalledAppService.go @@ -73,14 +73,6 @@ import ( "go.uber.org/zap" ) -const ( - DEFAULT_ENVIRONMENT_OR_NAMESPACE_OR_PROJECT = "devtron" - CLUSTER_COMPONENT_DIR_PATH = "/cluster/component" - HELM_RELEASE_STATUS_FAILED = "Failed" - HELM_RELEASE_STATUS_PROGRESSING = "Progressing" - HELM_RELEASE_STATUS_UNKNOWN = "Unknown" -) - // DB operation + chart group + nats msg consume(to be removed) type InstalledAppService interface { GetAll(filter *appStoreBean.AppStoreFilter) (openapi.AppList, error) @@ -354,6 +346,7 @@ func (impl InstalledAppServiceImpl) createChartGroupEntryObject(installAppVersio }, } } + func (impl InstalledAppServiceImpl) performDeployStageOnAcd(installedAppVersion *appStoreBean.InstallAppVersionDTO, ctx context.Context, userId int32) (*appStoreBean.InstallAppVersionDTO, error) { installedAppVersion.ACDAppName = fmt.Sprintf("%s-%s", installedAppVersion.AppName, installedAppVersion.Environment.Name) chartGitAttr := &util.ChartGitAttribute{} @@ -498,6 +491,7 @@ func (impl InstalledAppServiceImpl) performDeployStageOnAcd(installedAppVersion } return installedAppVersion, nil } + func (impl InstalledAppServiceImpl) performDeployStage(installedAppVersionId int, installedAppVersionHistoryId int, userId int32) (*appStoreBean.InstallAppVersionDTO, error) { ctx := context.Background() installedAppVersion, err := impl.appStoreDeploymentService.GetInstalledAppVersion(installedAppVersionId, userId) @@ -640,7 +634,7 @@ func (impl *InstalledAppServiceImpl) Subscribe() error { func (impl *InstalledAppServiceImpl) DeployDefaultChartOnCluster(bean *cluster2.ClusterBean, userId int32) (bool, error) { // STEP 1 - create environment with name "devton" impl.logger.Infow("STEP 1", "create environment for cluster component", bean) - envName := fmt.Sprintf("%d-%s", bean.Id, DEFAULT_ENVIRONMENT_OR_NAMESPACE_OR_PROJECT) + envName := fmt.Sprintf("%d-%s", bean.Id, appStoreBean.DEFAULT_ENVIRONMENT_OR_NAMESPACE_OR_PROJECT) env, err := impl.envService.FindOne(envName) if err != nil && err != pg.ErrNoRows { return false, err @@ -662,13 +656,13 @@ func (impl *InstalledAppServiceImpl) DeployDefaultChartOnCluster(bean *cluster2. // STEP 2 - create project with name "devtron" impl.logger.Info("STEP 2", "create project for cluster components") - t, err := impl.teamRepository.FindByTeamName(DEFAULT_ENVIRONMENT_OR_NAMESPACE_OR_PROJECT) + t, err := impl.teamRepository.FindByTeamName(appStoreBean.DEFAULT_ENVIRONMENT_OR_NAMESPACE_OR_PROJECT) if err != nil && err != pg.ErrNoRows { return false, err } if err == pg.ErrNoRows { t := &repository4.Team{ - Name: DEFAULT_ENVIRONMENT_OR_NAMESPACE_OR_PROJECT, + Name: appStoreBean.DEFAULT_ENVIRONMENT_OR_NAMESPACE_OR_PROJECT, Active: true, AuditLog: sql.AuditLog{CreatedBy: userId, CreatedOn: time.Now(), UpdatedOn: time.Now(), UpdatedBy: userId}, } @@ -681,13 +675,13 @@ func (impl *InstalledAppServiceImpl) DeployDefaultChartOnCluster(bean *cluster2. // STEP 3- read the input data from env variables impl.logger.Info("STEP 3", "read the input data from env variables") - charts := &ChartComponents{} - var chartComponents []*ChartComponent - if _, err := os.Stat(CLUSTER_COMPONENT_DIR_PATH); os.IsNotExist(err) { + charts := &appStoreBean.ChartComponents{} + var chartComponents []*appStoreBean.ChartComponent + if _, err := os.Stat(appStoreBean.CLUSTER_COMPONENT_DIR_PATH); os.IsNotExist(err) { impl.logger.Infow("default cluster component directory error", "cluster", bean.ClusterName, "err", err) return false, nil } else { - fileInfo, err := ioutil.ReadDir(CLUSTER_COMPONENT_DIR_PATH) + fileInfo, err := ioutil.ReadDir(appStoreBean.CLUSTER_COMPONENT_DIR_PATH) if err != nil { impl.logger.Errorw("DeployDefaultChartOnCluster, err while reading directory", "err", err) return false, err @@ -695,12 +689,12 @@ func (impl *InstalledAppServiceImpl) DeployDefaultChartOnCluster(bean *cluster2. for _, file := range fileInfo { impl.logger.Infow("file", "name", file.Name()) if strings.Contains(file.Name(), ".yaml") { - content, err := ioutil.ReadFile(fmt.Sprintf("%s/%s", CLUSTER_COMPONENT_DIR_PATH, file.Name())) + content, err := ioutil.ReadFile(fmt.Sprintf("%s/%s", appStoreBean.CLUSTER_COMPONENT_DIR_PATH, file.Name())) if err != nil { impl.logger.Errorw("DeployDefaultChartOnCluster, error on reading file", "err", err) return false, err } - chartComponent := &ChartComponent{ + chartComponent := &appStoreBean.ChartComponent{ Name: strings.ReplaceAll(file.Name(), ".yaml", ""), Values: string(content), } @@ -749,14 +743,6 @@ func (impl *InstalledAppServiceImpl) DeployDefaultChartOnCluster(bean *cluster2. return true, nil } -type ChartComponents struct { - ChartComponent []*ChartComponent `json:"charts"` -} -type ChartComponent struct { - Name string `json:"name"` - Values string `json:"values"` -} - func (impl InstalledAppServiceImpl) DeployDefaultComponent(chartGroupInstallRequest *chartGroup.ChartGroupInstallRequest) (*chartGroup.ChartGroupInstallAppRes, error) { impl.logger.Debugw("bulk app install request", "req", chartGroupInstallRequest) //save in db @@ -992,22 +978,22 @@ func (impl InstalledAppServiceImpl) getReleaseStatusFromHelmReleaseInstallStatus impl.logger.Errorw("error in unmarshalling helm release install status") return releaseStatus } - if status == HELM_RELEASE_STATUS_FAILED { + if status == appStoreBean.HELM_RELEASE_STATUS_FAILED { releaseStatus.Status = status releaseStatus.Description = helmInstallStatus.Message releaseStatus.Message = "Release install/upgrade failed" - } else if status == HELM_RELEASE_STATUS_PROGRESSING { + } else if status == appStoreBean.HELM_RELEASE_STATUS_PROGRESSING { releaseStatus.Status = status releaseStatus.Description = helmInstallStatus.Message releaseStatus.Message = helmInstallStatus.Message } else { // there can be a case when helm release is created but we are not able to fetch it - releaseStatus.Status = HELM_RELEASE_STATUS_UNKNOWN + releaseStatus.Status = appStoreBean.HELM_RELEASE_STATUS_UNKNOWN releaseStatus.Description = "Unable to fetch release for app" releaseStatus.Message = "Unable to fetch release for app" } } else { - releaseStatus.Status = HELM_RELEASE_STATUS_UNKNOWN + releaseStatus.Status = appStoreBean.HELM_RELEASE_STATUS_UNKNOWN releaseStatus.Description = "Release not found" releaseStatus.Message = "Release not found " } From 0eb82b73472cb379344e3acd402ede201063afb8 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Mon, 15 Jan 2024 18:48:55 +0530 Subject: [PATCH 10/64] removed: unused const --- pkg/appStore/bean/bean.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pkg/appStore/bean/bean.go b/pkg/appStore/bean/bean.go index 6254551ce7..314e0fd7f6 100644 --- a/pkg/appStore/bean/bean.go +++ b/pkg/appStore/bean/bean.go @@ -169,11 +169,6 @@ type Dependency struct { Repository string `json:"repository"` } -const BULK_APPSTORE_DEPLOY_TOPIC = "ORCHESTRATOR.APP-STORE.BULK-DEPLOY" -const BULK_APPSTORE_DEPLOY_GROUP = "ORCHESTRATOR.APP-STORE.BULK-DEPLOY-GROUP-1" - -const BULK_APPSTORE_DEPLOY_DURABLE = "ORCHESTRATOR.APP-STORE.BULK-DEPLOY.DURABLE-1" - type DeployPayload struct { InstalledAppVersionId int InstalledAppVersionHistoryId int From 765df77f07586978e874f07d24c579317d47bf2d Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Mon, 15 Jan 2024 19:05:11 +0530 Subject: [PATCH 11/64] review comments --- pkg/app/AppListingService.go | 10 +++--- pkg/bulkAction/BulkUpdateService.go | 2 +- pkg/chart/ChartService.go | 8 ++--- .../deployedAppMetrics/DeployedAppMetrics.go | 24 ++++++------- .../EnvLevelAppMetricsRepository.go | 2 +- .../DeploymentPipelineConfigService.go | 4 +-- pkg/pipeline/PropertiesConfig.go | 4 +-- wire_gen.go | 34 +++++++++---------- 8 files changed, 44 insertions(+), 44 deletions(-) diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index 462e851cef..0f67f2bad0 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -1672,11 +1672,11 @@ func (impl AppListingServiceImpl) FetchOtherEnvironment(ctx context.Context, app return envs, err } appLevelInfraMetrics := true //default val, not being derived from DB. TODO: remove this from FE since this is derived from prometheus config at cluster level and this logic is already present at FE - newCtx, span = otel.Tracer("appLevelMetricsRepository").Start(newCtx, "FindByAppId") - appLevelAppMetrics, err := impl.deployedAppMetricsService.GetMetricsFlagByAppIdEvenIfNotInDb(appId) + newCtx, span = otel.Tracer("deployedAppMetricsService").Start(newCtx, "GetMetricsFlagByAppId") + appLevelAppMetrics, err := impl.deployedAppMetricsService.GetMetricsFlagByAppId(appId) span.End() if err != nil { - impl.Logger.Errorw("error, GetMetricsFlagByAppIdEvenIfNotInDb", "err", err, "appId", appId) + impl.Logger.Errorw("error, GetMetricsFlagByAppId", "err", err, "appId", appId) return envs, err } newCtx, span = otel.Tracer("chartRepository").Start(newCtx, "FindLatestChartForAppByAppId") @@ -1714,9 +1714,9 @@ func (impl AppListingServiceImpl) FetchMinDetailOtherEnvironment(appId int) ([]* return envs, err } appLevelInfraMetrics := true //default val, not being derived from DB. TODO: remove this from FE since this is derived from prometheus config at cluster level and this logic is already present at FE - appLevelAppMetrics, err := impl.deployedAppMetricsService.GetMetricsFlagByAppIdEvenIfNotInDb(appId) + appLevelAppMetrics, err := impl.deployedAppMetricsService.GetMetricsFlagByAppId(appId) if err != nil { - impl.Logger.Errorw("error, GetMetricsFlagByAppIdEvenIfNotInDb", "err", err, "appId", appId) + impl.Logger.Errorw("error, GetMetricsFlagByAppId", "err", err, "appId", appId) return nil, err } diff --git a/pkg/bulkAction/BulkUpdateService.go b/pkg/bulkAction/BulkUpdateService.go index 9991ca32ee..d48612f705 100644 --- a/pkg/bulkAction/BulkUpdateService.go +++ b/pkg/bulkAction/BulkUpdateService.go @@ -453,7 +453,7 @@ func (impl BulkUpdateServiceImpl) BulkUpdateDeploymentTemplate(bulkUpdatePayload deploymentTemplateBulkUpdateResponse.Successful = append(deploymentTemplateBulkUpdateResponse.Successful, bulkUpdateSuccessResponse) //creating history entry for deployment template - appLevelAppMetricsEnabled, err := impl.deployedAppMetricsService.GetMetricsFlagByAppIdEvenIfNotInDb(chart.AppId) + appLevelAppMetricsEnabled, err := impl.deployedAppMetricsService.GetMetricsFlagByAppId(chart.AppId) if err != nil { impl.logger.Errorw("error in getting app level metrics app level", "error", err, "appId", chart.AppId) return nil diff --git a/pkg/chart/ChartService.go b/pkg/chart/ChartService.go index ceb5956051..d3219e27c3 100644 --- a/pkg/chart/ChartService.go +++ b/pkg/chart/ChartService.go @@ -444,7 +444,7 @@ func (impl ChartServiceImpl) Create(templateRequest TemplateRequest, ctx context ChartRefId: templateRequest.ChartRefId, UserId: templateRequest.UserId, } - err = impl.deployedAppMetricsService.CheckAndUpdateAppOrEnvLevelMetrics(ctx, appLevelMetricsUpdateReq) + err = impl.deployedAppMetricsService.CreateOrUpdateAppOrEnvLevelMetrics(ctx, appLevelMetricsUpdateReq) if err != nil { impl.logger.Errorw("error, CheckAndUpdateAppOrEnvLevelMetrics", "err", err, "req", appLevelMetricsUpdateReq) return nil, err @@ -727,7 +727,7 @@ func (impl ChartServiceImpl) FindLatestChartForAppByAppId(appId int) (chartTempl impl.logger.Errorw("error in fetching chart ", "appId", appId, "err", err) return nil, err } - isAppMetricsEnabled, err := impl.deployedAppMetricsService.GetMetricsFlagByAppIdEvenIfNotInDb(appId) + isAppMetricsEnabled, err := impl.deployedAppMetricsService.GetMetricsFlagByAppId(appId) if err != nil { impl.logger.Errorw("error in fetching app-metrics", "appId", appId, "err", err) return nil, err @@ -742,7 +742,7 @@ func (impl ChartServiceImpl) GetByAppIdAndChartRefId(appId int, chartRefId int) impl.logger.Errorw("error in fetching chart ", "appId", appId, "err", err) return nil, err } - isAppMetricsEnabled, err := impl.deployedAppMetricsService.GetMetricsFlagByAppIdEvenIfNotInDb(appId) + isAppMetricsEnabled, err := impl.deployedAppMetricsService.GetMetricsFlagByAppId(appId) if err != nil { impl.logger.Errorw("error in fetching app-metrics", "appId", appId, "err", err) return nil, err @@ -841,7 +841,7 @@ func (impl ChartServiceImpl) UpdateAppOverride(ctx context.Context, templateRequ ChartRefId: templateRequest.ChartRefId, UserId: templateRequest.UserId, } - err = impl.deployedAppMetricsService.CheckAndUpdateAppOrEnvLevelMetrics(ctx, appLevelMetricsUpdateReq) + err = impl.deployedAppMetricsService.CreateOrUpdateAppOrEnvLevelMetrics(ctx, appLevelMetricsUpdateReq) if err != nil { impl.logger.Errorw("error, CheckAndUpdateAppOrEnvLevelMetrics", "err", err, "req", appLevelMetricsUpdateReq) return nil, err diff --git a/pkg/deployment/manifest/deployedAppMetrics/DeployedAppMetrics.go b/pkg/deployment/manifest/deployedAppMetrics/DeployedAppMetrics.go index b57eb6dafe..2144b71cd5 100644 --- a/pkg/deployment/manifest/deployedAppMetrics/DeployedAppMetrics.go +++ b/pkg/deployment/manifest/deployedAppMetrics/DeployedAppMetrics.go @@ -14,10 +14,10 @@ import ( ) type DeployedAppMetricsService interface { - GetMetricsFlagByAppIdEvenIfNotInDb(appId int) (bool, error) + GetMetricsFlagByAppId(appId int) (bool, error) GetMetricsFlagByAppIdAndEnvId(appId, envId int) (bool, error) GetMetricsFlagForAPipelineByAppIdAndEnvId(appId, envId int) (bool, error) - CheckAndUpdateAppOrEnvLevelMetrics(ctx context.Context, req *bean.DeployedAppMetricsRequest) error + CreateOrUpdateAppOrEnvLevelMetrics(ctx context.Context, req *bean.DeployedAppMetricsRequest) error DeleteEnvLevelMetricsIfPresent(appId, envId int) error } @@ -40,7 +40,7 @@ func NewDeployedAppMetricsServiceImpl(logger *zap.SugaredLogger, } } -func (impl *DeployedAppMetricsServiceImpl) GetMetricsFlagByAppIdEvenIfNotInDb(appId int) (bool, error) { +func (impl *DeployedAppMetricsServiceImpl) GetMetricsFlagByAppId(appId int) (bool, error) { isAppMetricsEnabled := false appMetrics, err := impl.appLevelMetricsRepository.FindByAppId(appId) if err != nil && err != pg.ErrNoRows { @@ -56,12 +56,12 @@ func (impl *DeployedAppMetricsServiceImpl) GetMetricsFlagByAppIdEvenIfNotInDb(ap func (impl *DeployedAppMetricsServiceImpl) GetMetricsFlagByAppIdAndEnvId(appId, envId int) (bool, error) { isAppMetricsEnabled := false envLevelAppMetrics, err := impl.envLevelMetricsRepository.FindByAppIdAndEnvId(appId, envId) - if err != nil && err != pg.ErrNoRows { + if err != nil { impl.logger.Errorw("error in getting env level app metrics", "err", err, "appId", appId, "envId", envId) return isAppMetricsEnabled, err } if envLevelAppMetrics != nil { - isAppMetricsEnabled = *envLevelAppMetrics.AppMetrics + isAppMetricsEnabled = envLevelAppMetrics.AppMetrics } return isAppMetricsEnabled, nil } @@ -74,20 +74,20 @@ func (impl *DeployedAppMetricsServiceImpl) GetMetricsFlagForAPipelineByAppIdAndE impl.logger.Errorw("error in getting env level app metrics", "err", err, "appId", appId, "envId", envId) return isAppMetricsEnabled, err } else if err == pg.ErrNoRows { - isAppLevelMetricsEnabled, err := impl.GetMetricsFlagByAppIdEvenIfNotInDb(appId) + isAppLevelMetricsEnabled, err := impl.GetMetricsFlagByAppId(appId) if err != nil { - impl.logger.Errorw("error, GetMetricsFlagByAppIdEvenIfNotInDb", "err", err, "appId", appId) + impl.logger.Errorw("error, GetMetricsFlagByAppId", "err", err, "appId", appId) return false, err } isAppMetricsEnabled = isAppLevelMetricsEnabled - } else if envLevelAppMetrics != nil && envLevelAppMetrics.AppMetrics != nil { - isAppMetricsEnabled = *envLevelAppMetrics.AppMetrics + } else if envLevelAppMetrics != nil { + isAppMetricsEnabled = envLevelAppMetrics.AppMetrics } return isAppMetricsEnabled, nil } // CheckAndUpdateAppOrEnvLevelMetrics - this method checks whether chart being used supports metrics or not, is app level or env level and updates accordingly -func (impl *DeployedAppMetricsServiceImpl) CheckAndUpdateAppOrEnvLevelMetrics(ctx context.Context, req *bean.DeployedAppMetricsRequest) error { +func (impl *DeployedAppMetricsServiceImpl) CreateOrUpdateAppOrEnvLevelMetrics(ctx context.Context, req *bean.DeployedAppMetricsRequest) error { isAppMetricsSupported, err := impl.checkIsAppMetricsSupported(req.ChartRefId) if err != nil { return err @@ -189,7 +189,7 @@ func (impl *DeployedAppMetricsServiceImpl) createOrUpdateEnvLevelMetrics(req *be envLevelAppMetrics = &repository.EnvLevelAppMetrics{ AppId: req.AppId, EnvId: req.EnvId, - AppMetrics: &req.EnableMetrics, + AppMetrics: req.EnableMetrics, AuditLog: sql.AuditLog{ CreatedOn: time.Now(), UpdatedOn: time.Now(), @@ -203,7 +203,7 @@ func (impl *DeployedAppMetricsServiceImpl) createOrUpdateEnvLevelMetrics(req *be return nil, err } } else { - envLevelAppMetrics.AppMetrics = &req.EnableMetrics + envLevelAppMetrics.AppMetrics = req.EnableMetrics envLevelAppMetrics.UpdatedOn = time.Now() envLevelAppMetrics.UpdatedBy = req.UserId err = impl.envLevelMetricsRepository.Update(envLevelAppMetrics) diff --git a/pkg/deployment/manifest/deployedAppMetrics/repository/EnvLevelAppMetricsRepository.go b/pkg/deployment/manifest/deployedAppMetrics/repository/EnvLevelAppMetricsRepository.go index bb81432024..9c985ab6e3 100644 --- a/pkg/deployment/manifest/deployedAppMetrics/repository/EnvLevelAppMetricsRepository.go +++ b/pkg/deployment/manifest/deployedAppMetrics/repository/EnvLevelAppMetricsRepository.go @@ -28,7 +28,7 @@ type EnvLevelAppMetrics struct { Id int `sql:"id,pk"` AppId int `sql:"app_id,notnull"` EnvId int `sql:"env_id,notnull"` - AppMetrics *bool `sql:"app_metrics,notnull"` + AppMetrics bool `sql:"app_metrics,notnull"` //InfraMetrics *bool `sql:"infra_metrics,notnull"` not being used sql.AuditLog } diff --git a/pkg/pipeline/DeploymentPipelineConfigService.go b/pkg/pipeline/DeploymentPipelineConfigService.go index d7a3dfd307..794f7bfaac 100644 --- a/pkg/pipeline/DeploymentPipelineConfigService.go +++ b/pkg/pipeline/DeploymentPipelineConfigService.go @@ -1674,9 +1674,9 @@ func (impl *CdPipelineConfigServiceImpl) createCdPipeline(ctx context.Context, a } //getting global app metrics for cd pipeline create because env level metrics is not created yet appLevelAppMetricsEnabled := false - isAppLevelMetricsEnabled, err := impl.deployedAppMetricsService.GetMetricsFlagByAppIdEvenIfNotInDb(app.Id) + isAppLevelMetricsEnabled, err := impl.deployedAppMetricsService.GetMetricsFlagByAppId(app.Id) if err != nil { - impl.logger.Errorw("error, GetMetricsFlagByAppIdEvenIfNotInDb", "err", err, "appId", app.Id) + impl.logger.Errorw("error, GetMetricsFlagByAppId", "err", err, "appId", app.Id) return 0, err } appLevelAppMetricsEnabled = isAppLevelMetricsEnabled diff --git a/pkg/pipeline/PropertiesConfig.go b/pkg/pipeline/PropertiesConfig.go index 686042bd65..210bef4093 100644 --- a/pkg/pipeline/PropertiesConfig.go +++ b/pkg/pipeline/PropertiesConfig.go @@ -324,7 +324,7 @@ func (impl PropertiesConfigServiceImpl) UpdateEnvironmentProperties(appId int, p ChartRefId: oldEnvOverride.Chart.ChartRefId, UserId: propertiesRequest.UserId, } - err = impl.deployedAppMetricsService.CheckAndUpdateAppOrEnvLevelMetrics(context.Background(), envLevelMetricsUpdateReq) + err = impl.deployedAppMetricsService.CreateOrUpdateAppOrEnvLevelMetrics(context.Background(), envLevelMetricsUpdateReq) if err != nil { impl.logger.Errorw("error, CheckAndUpdateAppOrEnvLevelMetrics", "err", err, "req", envLevelMetricsUpdateReq) return nil, err @@ -434,7 +434,7 @@ func (impl PropertiesConfigServiceImpl) CreateIfRequired(chart *chartRepoReposit ChartRefId: chart.ChartRefId, UserId: userId, } - err = impl.deployedAppMetricsService.CheckAndUpdateAppOrEnvLevelMetrics(context.Background(), envLevelMetricsUpdateReq) + err = impl.deployedAppMetricsService.CreateOrUpdateAppOrEnvLevelMetrics(context.Background(), envLevelMetricsUpdateReq) if err != nil { impl.logger.Errorw("error, CheckAndUpdateAppOrEnvLevelMetrics", "err", err, "req", envLevelMetricsUpdateReq) return nil, isAppMetricsEnabled, err diff --git a/wire_gen.go b/wire_gen.go index 57c5566906..81d0d2ad50 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -45,7 +45,7 @@ import ( "github.com/devtron-labs/devtron/client/argocdServer/application" "github.com/devtron-labs/devtron/client/argocdServer/cluster" "github.com/devtron-labs/devtron/client/argocdServer/connection" - repository9 "github.com/devtron-labs/devtron/client/argocdServer/repository" + repository10 "github.com/devtron-labs/devtron/client/argocdServer/repository" "github.com/devtron-labs/devtron/client/cron" "github.com/devtron-labs/devtron/client/dashboard" "github.com/devtron-labs/devtron/client/events" @@ -62,7 +62,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" repository5 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" "github.com/devtron-labs/devtron/internal/sql/repository/helper" - repository13 "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging" + repository14 "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/sql/repository/resourceGroup" "github.com/devtron-labs/devtron/internal/sql/repository/security" @@ -103,7 +103,7 @@ import ( "github.com/devtron-labs/devtron/pkg/commonService" delete2 "github.com/devtron-labs/devtron/pkg/delete" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" - repository15 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/repository" + repository9 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/repository" "github.com/devtron-labs/devtron/pkg/deploymentGroup" "github.com/devtron-labs/devtron/pkg/devtronResource" repository8 "github.com/devtron-labs/devtron/pkg/devtronResource/repository" @@ -111,7 +111,7 @@ import ( "github.com/devtron-labs/devtron/pkg/externalLink" "github.com/devtron-labs/devtron/pkg/generateManifest" "github.com/devtron-labs/devtron/pkg/genericNotes" - repository10 "github.com/devtron-labs/devtron/pkg/genericNotes/repository" + repository11 "github.com/devtron-labs/devtron/pkg/genericNotes/repository" "github.com/devtron-labs/devtron/pkg/git" "github.com/devtron-labs/devtron/pkg/gitops" jira2 "github.com/devtron-labs/devtron/pkg/jira" @@ -120,7 +120,7 @@ import ( "github.com/devtron-labs/devtron/pkg/k8s/capacity" "github.com/devtron-labs/devtron/pkg/k8s/informer" "github.com/devtron-labs/devtron/pkg/kubernetesResourceAuditLogs" - repository14 "github.com/devtron-labs/devtron/pkg/kubernetesResourceAuditLogs/repository" + repository15 "github.com/devtron-labs/devtron/pkg/kubernetesResourceAuditLogs/repository" "github.com/devtron-labs/devtron/pkg/module" "github.com/devtron-labs/devtron/pkg/module/repo" "github.com/devtron-labs/devtron/pkg/module/store" @@ -129,10 +129,10 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/history" repository6 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" - repository11 "github.com/devtron-labs/devtron/pkg/pipeline/repository" + repository12 "github.com/devtron-labs/devtron/pkg/pipeline/repository" "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/plugin" - repository12 "github.com/devtron-labs/devtron/pkg/plugin/repository" + repository13 "github.com/devtron-labs/devtron/pkg/plugin/repository" "github.com/devtron-labs/devtron/pkg/projectManagementService/jira" resourceGroup2 "github.com/devtron-labs/devtron/pkg/resourceGroup" "github.com/devtron-labs/devtron/pkg/resourceQualifiers" @@ -359,8 +359,8 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - appLevelMetricsRepositoryImpl := repository15.NewAppLevelMetricsRepositoryImpl(db, sugaredLogger) - envLevelAppMetricsRepositoryImpl := repository15.NewEnvLevelAppMetricsRepositoryImpl(db, sugaredLogger) + appLevelMetricsRepositoryImpl := repository9.NewAppLevelMetricsRepositoryImpl(db, sugaredLogger) + envLevelAppMetricsRepositoryImpl := repository9.NewEnvLevelAppMetricsRepositoryImpl(db, sugaredLogger) deployedAppMetricsServiceImpl := deployedAppMetrics.NewDeployedAppMetricsServiceImpl(sugaredLogger, chartRefRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl) deploymentTemplateHistoryServiceImpl := history.NewDeploymentTemplateHistoryServiceImpl(sugaredLogger, deploymentTemplateHistoryRepositoryImpl, pipelineRepositoryImpl, chartRepositoryImpl, chartRefRepositoryImpl, userServiceImpl, cdWorkflowRepositoryImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl) chartWorkingDir := _wireChartWorkingDirValue @@ -375,7 +375,7 @@ func InitializeApp() (*App, error) { utilMergeUtil := util.MergeUtil{ Logger: sugaredLogger, } - repositoryServiceClientImpl := repository9.NewServiceClientImpl(sugaredLogger, argoCDConnectionManagerImpl) + repositoryServiceClientImpl := repository10.NewServiceClientImpl(sugaredLogger, argoCDConnectionManagerImpl) chartServiceImpl := chart.NewChartServiceImpl(chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, appRepositoryImpl, refChartDir, defaultChart, utilMergeUtil, repositoryServiceClientImpl, chartRefRepositoryImpl, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, httpClient, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl) devtronSecretConfig, err := util3.GetDevtronSecretName() if err != nil { @@ -388,8 +388,8 @@ func InitializeApp() (*App, error) { } pipelineStatusTimelineRepositoryImpl := pipelineConfig.NewPipelineStatusTimelineRepositoryImpl(db, sugaredLogger) appLabelRepositoryImpl := pipelineConfig.NewAppLabelRepositoryImpl(db) - genericNoteRepositoryImpl := repository10.NewGenericNoteRepositoryImpl(db) - genericNoteHistoryRepositoryImpl := repository10.NewGenericNoteHistoryRepositoryImpl(db) + genericNoteRepositoryImpl := repository11.NewGenericNoteRepositoryImpl(db) + genericNoteHistoryRepositoryImpl := repository11.NewGenericNoteHistoryRepositoryImpl(db) genericNoteHistoryServiceImpl := genericNotes.NewGenericNoteHistoryServiceImpl(genericNoteHistoryRepositoryImpl, sugaredLogger) genericNoteServiceImpl := genericNotes.NewGenericNoteServiceImpl(genericNoteRepositoryImpl, genericNoteHistoryServiceImpl, userRepositoryImpl, sugaredLogger) materialRepositoryImpl := pipelineConfig.NewMaterialRepositoryImpl(db) @@ -427,7 +427,7 @@ func InitializeApp() (*App, error) { } appStoreDeploymentServiceImpl := service.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, clusterInstalledAppsRepositoryImpl, appRepositoryImpl, appStoreDeploymentHelmServiceImpl, appStoreDeploymentArgoCdServiceImpl, environmentServiceImpl, clusterServiceImplExtended, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, globalEnvVariables, installedAppVersionHistoryRepositoryImpl, gitOpsConfigRepositoryImpl, attributesServiceImpl, deploymentServiceTypeConfig, chartTemplateServiceImpl, acdConfig) k8sCommonServiceImpl := k8s2.NewK8sCommonServiceImpl(sugaredLogger, k8sUtil, clusterServiceImplExtended) - manifestPushConfigRepositoryImpl := repository11.NewManifestPushConfigRepository(sugaredLogger, db) + manifestPushConfigRepositoryImpl := repository12.NewManifestPushConfigRepository(sugaredLogger, db) gitOpsManifestPushServiceImpl := app2.NewGitOpsManifestPushServiceImpl(sugaredLogger, chartTemplateServiceImpl, chartServiceImpl, gitOpsConfigRepositoryImpl, gitFactory, pipelineStatusTimelineServiceImpl, pipelineStatusTimelineRepositoryImpl, acdConfig) appServiceImpl := app2.NewAppService(envConfigOverrideRepositoryImpl, pipelineOverrideRepositoryImpl, mergeUtil, sugaredLogger, ciArtifactRepositoryImpl, pipelineRepositoryImpl, dbMigrationConfigRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, applicationServiceClientImpl, tokenCache, acdAuthConfig, enforcerImpl, enforcerUtilImpl, userServiceImpl, appListingRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, imageScanDeployInfoRepositoryImpl, imageScanHistoryRepositoryImpl, argoK8sClientImpl, gitFactory, pipelineStrategyHistoryServiceImpl, configMapHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, chartTemplateServiceImpl, refChartDir, chartRefRepositoryImpl, chartServiceImpl, helmAppClientImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, appCrudOperationServiceImpl, configMapHistoryRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, dockerRegistryIpsConfigServiceImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, gitOpsConfigRepositoryImpl, appStatusServiceImpl, installedAppRepositoryImpl, appStoreDeploymentServiceImpl, k8sCommonServiceImpl, installedAppVersionHistoryRepositoryImpl, globalEnvVariables, helmAppServiceImpl, manifestPushConfigRepositoryImpl, gitOpsManifestPushServiceImpl, argoClientWrapperServiceImpl, scopedVariableCMCSManagerImpl, acdConfig) validate, err := util.IntValidator() @@ -461,8 +461,8 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - pipelineStageRepositoryImpl := repository11.NewPipelineStageRepository(sugaredLogger, db) - globalPluginRepositoryImpl := repository12.NewGlobalPluginRepository(sugaredLogger, db) + pipelineStageRepositoryImpl := repository12.NewPipelineStageRepository(sugaredLogger, db) + globalPluginRepositoryImpl := repository13.NewGlobalPluginRepository(sugaredLogger, db) pipelineStageServiceImpl := pipeline.NewPipelineStageService(sugaredLogger, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, pipelineRepositoryImpl, scopedVariableManagerImpl) globalPluginServiceImpl := plugin.NewGlobalPluginService(sugaredLogger, globalPluginRepositoryImpl, pipelineStageRepositoryImpl) dockerRegistryConfigImpl := pipeline.NewDockerRegistryConfigImpl(sugaredLogger, helmAppServiceImpl, dockerArtifactStoreRepositoryImpl, dockerRegistryIpsConfigRepositoryImpl, ociRegistryConfigRepositoryImpl) @@ -500,7 +500,7 @@ func InitializeApp() (*App, error) { buildPipelineSwitchServiceImpl := pipeline.NewBuildPipelineSwitchServiceImpl(sugaredLogger, ciPipelineRepositoryImpl, ciCdPipelineOrchestratorImpl, pipelineRepositoryImpl, ciWorkflowRepositoryImpl, appWorkflowRepositoryImpl, ciPipelineHistoryServiceImpl, ciTemplateOverrideRepositoryImpl, ciPipelineMaterialRepositoryImpl) ciPipelineConfigServiceImpl := pipeline.NewCiPipelineConfigServiceImpl(sugaredLogger, ciCdPipelineOrchestratorImpl, dockerArtifactStoreRepositoryImpl, materialRepositoryImpl, appRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ecrConfig, appWorkflowRepositoryImpl, ciCdConfig, attributesServiceImpl, pipelineStageServiceImpl, ciPipelineMaterialRepositoryImpl, ciTemplateServiceImpl, ciTemplateOverrideRepositoryImpl, ciTemplateHistoryServiceImpl, enforcerUtilImpl, ciWorkflowRepositoryImpl, resourceGroupServiceImpl, customTagServiceImpl, ciPipelineHistoryServiceImpl, cdWorkflowRepositoryImpl, buildPipelineSwitchServiceImpl) ciMaterialConfigServiceImpl := pipeline.NewCiMaterialConfigServiceImpl(sugaredLogger, materialRepositoryImpl, ciTemplateServiceImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, gitMaterialHistoryServiceImpl, pipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl) - imageTaggingRepositoryImpl := repository13.NewImageTaggingRepositoryImpl(db) + imageTaggingRepositoryImpl := repository14.NewImageTaggingRepositoryImpl(db) imageTaggingServiceImpl := pipeline.NewImageTaggingServiceImpl(imageTaggingRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, sugaredLogger) propertiesConfigServiceImpl := pipeline.NewPropertiesConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, chartRefRepositoryImpl, utilMergeUtil, environmentRepositoryImpl, ciCdPipelineOrchestratorImpl, applicationServiceClientImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl) pipelineDeploymentServiceTypeConfig, err := pipeline.GetDeploymentServiceTypeConfig() @@ -551,7 +551,7 @@ func InitializeApp() (*App, error) { migrateDbRouterImpl := router.NewMigrateDbRouterImpl(migrateDbRestHandlerImpl) appStoreVersionValuesRepositoryImpl := appStoreValuesRepository.NewAppStoreVersionValuesRepositoryImpl(sugaredLogger, db) appStoreValuesServiceImpl := service2.NewAppStoreValuesServiceImpl(sugaredLogger, appStoreApplicationVersionRepositoryImpl, installedAppRepositoryImpl, appStoreVersionValuesRepositoryImpl, userServiceImpl) - k8sResourceHistoryRepositoryImpl := repository14.NewK8sResourceHistoryRepositoryImpl(db, sugaredLogger) + k8sResourceHistoryRepositoryImpl := repository15.NewK8sResourceHistoryRepositoryImpl(db, sugaredLogger) k8sResourceHistoryServiceImpl := kubernetesResourceAuditLogs.Newk8sResourceHistoryServiceImpl(k8sResourceHistoryRepositoryImpl, sugaredLogger, appRepositoryImpl, environmentRepositoryImpl) ephemeralContainersRepositoryImpl := repository2.NewEphemeralContainersRepositoryImpl(db) ephemeralContainerServiceImpl := cluster2.NewEphemeralContainerServiceImpl(ephemeralContainersRepositoryImpl, sugaredLogger) From 5038a0b2ef48a02130876159553d1d2c524f62da Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Mon, 15 Jan 2024 19:35:57 +0530 Subject: [PATCH 12/64] migrated some methods from chartService to chartRefService --- api/deployment/DeploymentConfigRestHandler.go | 2 +- .../app/DeploymentPipelineRestHandler.go | 4 +-- pkg/chart/ChartService.go | 33 ----------------- .../chartRef/ChartRefService.go | 35 +++++++++++++++++++ 4 files changed, 38 insertions(+), 36 deletions(-) diff --git a/api/deployment/DeploymentConfigRestHandler.go b/api/deployment/DeploymentConfigRestHandler.go index d9d819959e..8e27a52a5d 100644 --- a/api/deployment/DeploymentConfigRestHandler.go +++ b/api/deployment/DeploymentConfigRestHandler.go @@ -85,7 +85,7 @@ func (handler *DeploymentConfigRestHandlerImpl) CreateChartFromFile(w http.Respo return } - err = handler.chartService.ValidateUploadedFileFormat(fileHeader.Filename) + err = handler.chartRefService.ValidateCustomChartUploadedFileFormat(fileHeader.Filename) if err != nil { handler.Logger.Errorw("request err, Unsupported format", "err", err, "payload", file) common.WriteJsonResp(w, errors.New("Unsupported format file is uploaded, please upload file with .tgz extension"), nil, http.StatusBadRequest) diff --git a/api/restHandler/app/DeploymentPipelineRestHandler.go b/api/restHandler/app/DeploymentPipelineRestHandler.go index 8889f2d5ea..5e9489f69a 100644 --- a/api/restHandler/app/DeploymentPipelineRestHandler.go +++ b/api/restHandler/app/DeploymentPipelineRestHandler.go @@ -877,7 +877,7 @@ func (handler PipelineConfigRestHandlerImpl) GetEnvConfigOverride(w http.Respons return } - schema, readme, err := handler.chartService.GetSchemaAndReadmeForTemplateByChartRefId(chartRefId) + schema, readme, err := handler.chartRefService.GetSchemaAndReadmeForTemplateByChartRefId(chartRefId) if err != nil { handler.Logger.Errorw("err in getting schema and readme, GetEnvConfigOverride", "err", err, "appId", appId, "chartRefId", chartRefId) } @@ -997,7 +997,7 @@ func (handler PipelineConfigRestHandlerImpl) GetDeploymentTemplate(w http.Respon return } - schema, readme, err := handler.chartService.GetSchemaAndReadmeForTemplateByChartRefId(chartRefId) + schema, readme, err := handler.chartRefService.GetSchemaAndReadmeForTemplateByChartRefId(chartRefId) if err != nil { handler.Logger.Errorw("err in getting schema and readme, GetDeploymentTemplate", "err", err, "appId", appId, "chartRefId", chartRefId) } diff --git a/pkg/chart/ChartService.go b/pkg/chart/ChartService.go index 0212bc570a..dca2a43761 100644 --- a/pkg/chart/ChartService.go +++ b/pkg/chart/ChartService.go @@ -72,8 +72,6 @@ type ChartService interface { UpgradeForApp(appId int, chartRefId int, newAppOverride map[string]interface{}, userId int32, ctx context.Context) (bool, error) DeploymentTemplateValidate(ctx context.Context, templatejson interface{}, chartRefId int, scope resourceQualifiers.Scope) (bool, error) JsonSchemaExtractFromFile(chartRefId int) (map[string]interface{}, string, error) - GetSchemaAndReadmeForTemplateByChartRefId(chartRefId int) (schema []byte, readme []byte, err error) - ValidateUploadedFileFormat(fileName string) error CheckIfChartRefUserUploadedByAppId(id int) (bool, error) PatchEnvOverrides(values json.RawMessage, oldChartType string, newChartType string) (json.RawMessage, error) FlaggerCanaryEnabled(values json.RawMessage) (bool, error) @@ -151,30 +149,6 @@ func (impl ChartServiceImpl) PatchEnvOverrides(values json.RawMessage, oldChartT return PatchWinterSoldierConfig(values, newChartType) } -func (impl ChartServiceImpl) GetSchemaAndReadmeForTemplateByChartRefId(chartRefId int) ([]byte, []byte, error) { - refChart, _, err, _, _ := impl.chartRefService.GetRefChart(chartRefId) - if err != nil { - impl.logger.Errorw("error in getting refChart", "err", err, "chartRefId", chartRefId) - return nil, nil, err - } - var schemaByte []byte - var readmeByte []byte - err = impl.chartRefService.CheckChartExists(chartRefId) - if err != nil { - impl.logger.Errorw("error in getting refChart", "err", err, "chartRefId", chartRefId) - return nil, nil, err - } - schemaByte, err = ioutil.ReadFile(filepath.Clean(filepath.Join(refChart, "schema.json"))) - if err != nil { - impl.logger.Errorw("error in reading schema.json file for refChart", "err", err, "chartRefId", chartRefId) - } - readmeByte, err = ioutil.ReadFile(filepath.Clean(filepath.Join(refChart, "README.md"))) - if err != nil { - impl.logger.Errorw("error in reading readme file for refChart", "err", err, "chartRefId", chartRefId) - } - return schemaByte, readmeByte, nil -} - func (impl ChartServiceImpl) GetAppOverrideForDefaultTemplate(chartRefId int) (map[string]interface{}, string, error) { err := impl.chartRefService.CheckChartExists(chartRefId) if err != nil { @@ -1063,13 +1037,6 @@ func (impl ChartServiceImpl) JsonSchemaExtractFromFile(chartRefId int) (map[stri } } -func (impl *ChartServiceImpl) ValidateUploadedFileFormat(fileName string) error { - if !strings.HasSuffix(fileName, ".tgz") { - return errors.New("unsupported format") - } - return nil -} - func (impl ChartServiceImpl) CheckIfChartRefUserUploadedByAppId(id int) (bool, error) { chartInfo, err := impl.chartRepository.FindLatestChartForAppByAppId(id) if err != nil { diff --git a/pkg/deployment/manifest/deploymentTemplate/chartRef/ChartRefService.go b/pkg/deployment/manifest/deploymentTemplate/chartRef/ChartRefService.go index 0bfd0bd278..01afd15aff 100644 --- a/pkg/deployment/manifest/deploymentTemplate/chartRef/ChartRefService.go +++ b/pkg/deployment/manifest/deploymentTemplate/chartRef/ChartRefService.go @@ -33,6 +33,9 @@ type ChartRefService interface { SaveCustomChart(req *bean.CustomChartRefDto) error FetchCustomChartsInfo() ([]*bean.ChartDto, error) + ValidateCustomChartUploadedFileFormat(fileName string) error + + GetSchemaAndReadmeForTemplateByChartRefId(chartRefId int) ([]byte, []byte, error) ChartRefIdsCompatible(oldChartRefId int, newChartRefId int) (bool, string, string) @@ -61,6 +64,13 @@ func NewChartRefServiceImpl(logger *zap.SugaredLogger, } } +func (impl *ChartRefServiceImpl) ValidateCustomChartUploadedFileFormat(fileName string) error { + if !strings.HasSuffix(fileName, ".tgz") { + return errors.New("unsupported format") + } + return nil +} + func (impl *ChartRefServiceImpl) GetDefault() (*bean.ChartRefDto, error) { chartRef, err := impl.chartRefRepository.GetDefault() if err != nil { @@ -98,6 +108,7 @@ func (impl *ChartRefServiceImpl) GetAllChartMetadata() (map[string]bean.ChartRef } return chartsMetadataMap, nil } + func (impl *ChartRefServiceImpl) ChartRefIdsCompatible(oldChartRefId int, newChartRefId int) (bool, string, string) { oldChart, err := impl.FindById(oldChartRefId) if err != nil { @@ -199,6 +210,30 @@ func (impl *ChartRefServiceImpl) GetRefChart(chartRefId int) (string, string, er return chartPath, template, nil, version, pipelineStrategyPath } +func (impl *ChartRefServiceImpl) GetSchemaAndReadmeForTemplateByChartRefId(chartRefId int) ([]byte, []byte, error) { + refChart, _, err, _, _ := impl.GetRefChart(chartRefId) + if err != nil { + impl.logger.Errorw("error in getting refChart", "err", err, "chartRefId", chartRefId) + return nil, nil, err + } + var schemaByte []byte + var readmeByte []byte + err = impl.CheckChartExists(chartRefId) + if err != nil { + impl.logger.Errorw("error in getting refChart", "err", err, "chartRefId", chartRefId) + return nil, nil, err + } + schemaByte, err = ioutil.ReadFile(filepath.Clean(filepath.Join(refChart, "schema.json"))) + if err != nil { + impl.logger.Errorw("error in reading schema.json file for refChart", "err", err, "chartRefId", chartRefId) + } + readmeByte, err = ioutil.ReadFile(filepath.Clean(filepath.Join(refChart, "README.md"))) + if err != nil { + impl.logger.Errorw("error in reading readme file for refChart", "err", err, "chartRefId", chartRefId) + } + return schemaByte, readmeByte, nil +} + func (impl *ChartRefServiceImpl) ChartRefAutocomplete() ([]*bean.ChartRefAutocompleteDto, error) { var chartRefs []*bean.ChartRefAutocompleteDto results, err := impl.chartRefRepository.GetAll() From c38d5288e869569407ca09aa354f738792c99026 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Mon, 15 Jan 2024 20:09:41 +0530 Subject: [PATCH 13/64] added dt validation service interface --- .../DeploymentTemplateValidationService.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 pkg/deployment/manifest/deploymentTemplate/DeploymentTemplateValidationService.go diff --git a/pkg/deployment/manifest/deploymentTemplate/DeploymentTemplateValidationService.go b/pkg/deployment/manifest/deploymentTemplate/DeploymentTemplateValidationService.go new file mode 100644 index 0000000000..6c851bc518 --- /dev/null +++ b/pkg/deployment/manifest/deploymentTemplate/DeploymentTemplateValidationService.go @@ -0,0 +1,16 @@ +package deploymentTemplate + +import "go.uber.org/zap" + +type DeploymentTemplateValidationService interface { +} + +type DeploymentTemplateValidationServiceImpl struct { + logger *zap.SugaredLogger +} + +func NewDeploymentTemplateValidationServiceImpl(logger *zap.SugaredLogger) *DeploymentTemplateValidationServiceImpl { + return &DeploymentTemplateValidationServiceImpl{ + logger: logger, + } +} From f0e449e6cfba75c77028b077d1ef33373dbe5d87 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 16 Jan 2024 17:41:25 +0530 Subject: [PATCH 14/64] minor refactoring --- .../app/DeploymentPipelineRestHandler.go | 8 +- pkg/chart/ChartService.go | 98 +---------------- .../chartRef/ChartRefService.go | 100 +++++++++++++++++- .../DeployementTemplateService.go | 2 +- wire_gen.go | 8 +- 5 files changed, 109 insertions(+), 107 deletions(-) diff --git a/api/restHandler/app/DeploymentPipelineRestHandler.go b/api/restHandler/app/DeploymentPipelineRestHandler.go index 5e9489f69a..22eec8e413 100644 --- a/api/restHandler/app/DeploymentPipelineRestHandler.go +++ b/api/restHandler/app/DeploymentPipelineRestHandler.go @@ -1010,7 +1010,7 @@ func (handler PipelineConfigRestHandlerImpl) GetDeploymentTemplate(w http.Respon } if pg.ErrNoRows == err { - appOverride, _, err := handler.chartService.GetAppOverrideForDefaultTemplate(chartRefId) + appOverride, _, err := handler.chartRefService.GetAppOverrideForDefaultTemplate(chartRefId) if err != nil { handler.Logger.Errorw("service err, GetDeploymentTemplate", "err", err, "appId", appId, "chartRefId", chartRefId) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) @@ -1085,7 +1085,7 @@ func (handler *PipelineConfigRestHandlerImpl) GetDefaultDeploymentTemplate(w htt common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "unauthorized user", http.StatusForbidden) return } - defaultTemplate, _, err := handler.chartService.GetAppOverrideForDefaultTemplate(chartRefId) + defaultTemplate, _, err := handler.chartRefService.GetAppOverrideForDefaultTemplate(chartRefId) if err != nil { handler.Logger.Errorw("error in getting default deployment template, GetDefaultDeploymentTemplate", "err", err, "appId", appId, "chartRefId", chartRefId) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) @@ -1363,7 +1363,7 @@ func (handler PipelineConfigRestHandlerImpl) GetAppOverrideForDefaultTemplate(w } //RBAC - appOverride, _, err := handler.chartService.GetAppOverrideForDefaultTemplate(chartRefId) + appOverride, _, err := handler.chartRefService.GetAppOverrideForDefaultTemplate(chartRefId) if err != nil { handler.Logger.Errorw("service err, UpdateCiTemplate", "err", err, "appId", appId, "chartRefId", chartRefId) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) @@ -2311,7 +2311,7 @@ func (handler PipelineConfigRestHandlerImpl) UpgradeForAllApps(w http.ResponseWr return } - newAppOverride, _, err := handler.chartService.GetAppOverrideForDefaultTemplate(chartUpgradeRequest.ChartRefId) + newAppOverride, _, err := handler.chartRefService.GetAppOverrideForDefaultTemplate(chartUpgradeRequest.ChartRefId) if err != nil { handler.Logger.Errorw("service err, UpgradeForAllApps", "err", err, "payload", chartUpgradeRequest) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) diff --git a/pkg/chart/ChartService.go b/pkg/chart/ChartService.go index dca2a43761..3e32659efb 100644 --- a/pkg/chart/ChartService.go +++ b/pkg/chart/ChartService.go @@ -39,8 +39,6 @@ import ( chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" "github.com/devtron-labs/devtron/pkg/pipeline/history" - "io/ioutil" - "os" "path/filepath" "strconv" "strings" @@ -65,13 +63,11 @@ type ChartService interface { CreateChartFromEnvOverride(templateRequest TemplateRequest, ctx context.Context) (chart *TemplateRequest, err error) FindLatestChartForAppByAppId(appId int) (chartTemplate *TemplateRequest, err error) GetByAppIdAndChartRefId(appId int, chartRefId int) (chartTemplate *TemplateRequest, err error) - GetAppOverrideForDefaultTemplate(chartRefId int) (map[string]interface{}, string, error) UpdateAppOverride(ctx context.Context, templateRequest *TemplateRequest) (*TemplateRequest, error) IsReadyToTrigger(appId int, envId int, pipelineId int) (IsReady, error) FindPreviousChartByAppId(appId int) (chartTemplate *TemplateRequest, err error) UpgradeForApp(appId int, chartRefId int, newAppOverride map[string]interface{}, userId int32, ctx context.Context) (bool, error) DeploymentTemplateValidate(ctx context.Context, templatejson interface{}, chartRefId int, scope resourceQualifiers.Scope) (bool, error) - JsonSchemaExtractFromFile(chartRefId int) (map[string]interface{}, string, error) CheckIfChartRefUserUploadedByAppId(id int) (bool, error) PatchEnvOverrides(values json.RawMessage, oldChartType string, newChartType string) (json.RawMessage, error) FlaggerCanaryEnabled(values json.RawMessage) (bool, error) @@ -149,60 +145,6 @@ func (impl ChartServiceImpl) PatchEnvOverrides(values json.RawMessage, oldChartT return PatchWinterSoldierConfig(values, newChartType) } -func (impl ChartServiceImpl) GetAppOverrideForDefaultTemplate(chartRefId int) (map[string]interface{}, string, error) { - err := impl.chartRefService.CheckChartExists(chartRefId) - if err != nil { - impl.logger.Errorw("error in getting missing chart for chartRefId", "err", err, "chartRefId") - return nil, "", err - } - - refChart, _, err, _, _ := impl.chartRefService.GetRefChart(chartRefId) - if err != nil { - return nil, "", err - } - var appOverrideByte, envOverrideByte []byte - appOverrideByte, err = ioutil.ReadFile(filepath.Clean(filepath.Join(refChart, "app-values.yaml"))) - if err != nil { - impl.logger.Infow("App values yaml file is missing") - } else { - appOverrideByte, err = yaml.YAMLToJSON(appOverrideByte) - if err != nil { - return nil, "", err - } - } - - envOverrideByte, err = ioutil.ReadFile(filepath.Clean(filepath.Join(refChart, "env-values.yaml"))) - if err != nil { - impl.logger.Infow("Env values yaml file is missing") - } else { - envOverrideByte, err = yaml.YAMLToJSON(envOverrideByte) - if err != nil { - return nil, "", err - } - } - - messages := make(map[string]interface{}) - var merged []byte - if appOverrideByte == nil && envOverrideByte == nil { - return messages, "", nil - } else if appOverrideByte == nil || envOverrideByte == nil { - if appOverrideByte == nil { - merged = envOverrideByte - } else { - merged = appOverrideByte - } - } else { - merged, err = impl.mergeUtil.JsonPatch(appOverrideByte, []byte(envOverrideByte)) - if err != nil { - return nil, "", err - } - } - - appOverride := json.RawMessage(merged) - messages["defaultAppOverride"] = appOverride - return messages, string(merged), nil -} - type AppMetricsEnabled struct { AppMetrics bool `json:"app-metrics"` } @@ -924,7 +866,7 @@ const memory = "memory" func (impl ChartServiceImpl) DeploymentTemplateValidate(ctx context.Context, template interface{}, chartRefId int, scope resourceQualifiers.Scope) (bool, error) { _, span := otel.Tracer("orchestrator").Start(ctx, "JsonSchemaExtractFromFile") - schemajson, version, err := impl.JsonSchemaExtractFromFile(chartRefId) + schemajson, version, err := impl.chartRefService.JsonSchemaExtractFromFile(chartRefId) span.End() if err != nil { impl.logger.Errorw("Json Schema not found err, FindJsonSchema", "err", err) @@ -999,44 +941,6 @@ func (impl ChartServiceImpl) DeploymentTemplateValidate(ctx context.Context, tem } } -func (impl ChartServiceImpl) JsonSchemaExtractFromFile(chartRefId int) (map[string]interface{}, string, error) { - err := impl.chartRefService.CheckChartExists(chartRefId) - if err != nil { - impl.logger.Errorw("refChartDir Not Found", "err", err) - return nil, "", err - } - - refChartDir, _, err, version, _ := impl.chartRefService.GetRefChart(chartRefId) - if err != nil { - impl.logger.Errorw("refChartDir Not Found err, JsonSchemaExtractFromFile", err) - return nil, "", err - } - fileStatus := filepath.Join(refChartDir, "schema.json") - if _, err := os.Stat(fileStatus); os.IsNotExist(err) { - impl.logger.Errorw("Schema File Not Found err, JsonSchemaExtractFromFile", err) - return nil, "", err - } else { - jsonFile, err := os.Open(fileStatus) - if err != nil { - impl.logger.Errorw("jsonfile open err, JsonSchemaExtractFromFile", "err", err) - return nil, "", err - } - byteValueJsonFile, err := ioutil.ReadAll(jsonFile) - if err != nil { - impl.logger.Errorw("byteValueJsonFile read err, JsonSchemaExtractFromFile", "err", err) - return nil, "", err - } - - var schemajson map[string]interface{} - err = json.Unmarshal([]byte(byteValueJsonFile), &schemajson) - if err != nil { - impl.logger.Errorw("Unmarshal err in byteValueJsonFile, DeploymentTemplateValidate", "err", err) - return nil, "", err - } - return schemajson, version, nil - } -} - func (impl ChartServiceImpl) CheckIfChartRefUserUploadedByAppId(id int) (bool, error) { chartInfo, err := impl.chartRepository.FindLatestChartForAppByAppId(id) if err != nil { diff --git a/pkg/deployment/manifest/deploymentTemplate/chartRef/ChartRefService.go b/pkg/deployment/manifest/deploymentTemplate/chartRef/ChartRefService.go index 01afd15aff..2b10d904b6 100644 --- a/pkg/deployment/manifest/deploymentTemplate/chartRef/ChartRefService.go +++ b/pkg/deployment/manifest/deploymentTemplate/chartRef/ChartRefService.go @@ -2,6 +2,7 @@ package chartRef import ( "bytes" + "encoding/json" "errors" "fmt" "github.com/devtron-labs/devtron/internal/constants" @@ -35,6 +36,9 @@ type ChartRefService interface { FetchCustomChartsInfo() ([]*bean.ChartDto, error) ValidateCustomChartUploadedFileFormat(fileName string) error + GetAppOverrideForDefaultTemplate(chartRefId int) (map[string]interface{}, string, error) + + JsonSchemaExtractFromFile(chartRefId int) (map[string]interface{}, string, error) GetSchemaAndReadmeForTemplateByChartRefId(chartRefId int) ([]byte, []byte, error) ChartRefIdsCompatible(oldChartRefId int, newChartRefId int) (bool, string, string) @@ -49,11 +53,13 @@ type ChartRefServiceImpl struct { logger *zap.SugaredLogger chartRefRepository chartRepoRepository.ChartRefRepository chartTemplateService util.ChartTemplateService + mergeUtil util.MergeUtil } func NewChartRefServiceImpl(logger *zap.SugaredLogger, chartRefRepository chartRepoRepository.ChartRefRepository, - chartTemplateService util.ChartTemplateService) *ChartRefServiceImpl { + chartTemplateService util.ChartTemplateService, + mergeUtil util.MergeUtil) *ChartRefServiceImpl { // cache devtron reference charts list devtronChartList, _ := chartRefRepository.FetchAllChartInfoByUploadFlag(false) setReservedChartList(devtronChartList) @@ -61,6 +67,7 @@ func NewChartRefServiceImpl(logger *zap.SugaredLogger, logger: logger, chartRefRepository: chartRefRepository, chartTemplateService: chartTemplateService, + mergeUtil: mergeUtil, } } @@ -328,6 +335,97 @@ func (impl *ChartRefServiceImpl) CheckChartExists(chartRefId int) error { } return nil } +func (impl *ChartRefServiceImpl) GetAppOverrideForDefaultTemplate(chartRefId int) (map[string]interface{}, string, error) { + err := impl.CheckChartExists(chartRefId) + if err != nil { + impl.logger.Errorw("error in getting missing chart for chartRefId", "err", err, "chartRefId") + return nil, "", err + } + + refChart, _, err, _, _ := impl.GetRefChart(chartRefId) + if err != nil { + return nil, "", err + } + var appOverrideByte, envOverrideByte []byte + appOverrideByte, err = ioutil.ReadFile(filepath.Clean(filepath.Join(refChart, "app-values.yaml"))) + if err != nil { + impl.logger.Infow("App values yaml file is missing") + } else { + appOverrideByte, err = yaml.YAMLToJSON(appOverrideByte) + if err != nil { + return nil, "", err + } + } + + envOverrideByte, err = ioutil.ReadFile(filepath.Clean(filepath.Join(refChart, "env-values.yaml"))) + if err != nil { + impl.logger.Infow("Env values yaml file is missing") + } else { + envOverrideByte, err = yaml.YAMLToJSON(envOverrideByte) + if err != nil { + return nil, "", err + } + } + + messages := make(map[string]interface{}) + var merged []byte + if appOverrideByte == nil && envOverrideByte == nil { + return messages, "", nil + } else if appOverrideByte == nil || envOverrideByte == nil { + if appOverrideByte == nil { + merged = envOverrideByte + } else { + merged = appOverrideByte + } + } else { + merged, err = impl.mergeUtil.JsonPatch(appOverrideByte, []byte(envOverrideByte)) + if err != nil { + return nil, "", err + } + } + + appOverride := json.RawMessage(merged) + messages["defaultAppOverride"] = appOverride + return messages, string(merged), nil +} + +func (impl *ChartRefServiceImpl) JsonSchemaExtractFromFile(chartRefId int) (map[string]interface{}, string, error) { + err := impl.CheckChartExists(chartRefId) + if err != nil { + impl.logger.Errorw("refChartDir Not Found", "err", err) + return nil, "", err + } + + refChartDir, _, err, version, _ := impl.GetRefChart(chartRefId) + if err != nil { + impl.logger.Errorw("refChartDir Not Found err, JsonSchemaExtractFromFile", err) + return nil, "", err + } + fileStatus := filepath.Join(refChartDir, "schema.json") + if _, err := os.Stat(fileStatus); os.IsNotExist(err) { + impl.logger.Errorw("Schema File Not Found err, JsonSchemaExtractFromFile", err) + return nil, "", err + } else { + jsonFile, err := os.Open(fileStatus) + if err != nil { + impl.logger.Errorw("jsonfile open err, JsonSchemaExtractFromFile", "err", err) + return nil, "", err + } + byteValueJsonFile, err := ioutil.ReadAll(jsonFile) + if err != nil { + impl.logger.Errorw("byteValueJsonFile read err, JsonSchemaExtractFromFile", "err", err) + return nil, "", err + } + + var schemajson map[string]interface{} + err = json.Unmarshal([]byte(byteValueJsonFile), &schemajson) + if err != nil { + impl.logger.Errorw("Unmarshal err in byteValueJsonFile, DeploymentTemplateValidate", "err", err) + return nil, "", err + } + return schemajson, version, nil + } +} func (impl *ChartRefServiceImpl) ExtractChartIfMissing(chartData []byte, refChartDir string, location string) (*bean.ChartDataInfo, error) { binaryDataReader := bytes.NewReader(chartData) diff --git a/pkg/generateManifest/DeployementTemplateService.go b/pkg/generateManifest/DeployementTemplateService.go index 1cb3023bf7..b35924856a 100644 --- a/pkg/generateManifest/DeployementTemplateService.go +++ b/pkg/generateManifest/DeployementTemplateService.go @@ -200,7 +200,7 @@ func (impl DeploymentTemplateServiceImpl) GetDeploymentTemplate(ctx context.Cont } else { switch request.Type { case repository.DefaultVersions: - _, values, err = impl.chartService.GetAppOverrideForDefaultTemplate(request.ChartRefId) + _, values, err = impl.chartRefService.GetAppOverrideForDefaultTemplate(request.ChartRefId) resolvedValue = values case repository.PublishedOnEnvironments: values, resolvedValue, variableSnapshot, err = impl.fetchResolvedTemplateForPublishedEnvs(ctx, request) diff --git a/wire_gen.go b/wire_gen.go index f32319b90d..5e774bbe7a 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -367,7 +367,10 @@ func InitializeApp() (*App, error) { return nil, err } chartRefRepositoryImpl := chartRepoRepository.NewChartRefRepositoryImpl(db) - chartRefServiceImpl := chartRef.NewChartRefServiceImpl(sugaredLogger, chartRefRepositoryImpl, chartTemplateServiceImpl) + utilMergeUtil := util.MergeUtil{ + Logger: sugaredLogger, + } + chartRefServiceImpl := chartRef.NewChartRefServiceImpl(sugaredLogger, chartRefRepositoryImpl, chartTemplateServiceImpl, utilMergeUtil) appServiceImpl := app2.NewAppService(envConfigOverrideRepositoryImpl, pipelineOverrideRepositoryImpl, mergeUtil, sugaredLogger, pipelineRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, applicationServiceClientImpl, appRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, chartTemplateServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, appStatusServiceImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, globalEnvVariables, scopedVariableCMCSManagerImpl, acdConfig, chartRefServiceImpl) validate, err := util.IntValidator() if err != nil { @@ -462,9 +465,6 @@ func InitializeApp() (*App, error) { ciBuildConfigServiceImpl := pipeline.NewCiBuildConfigServiceImpl(sugaredLogger, ciBuildConfigRepositoryImpl) ciTemplateServiceImpl := pipeline.NewCiTemplateServiceImpl(sugaredLogger, ciBuildConfigServiceImpl, ciTemplateRepositoryImpl, ciTemplateOverrideRepositoryImpl) chartRepoRepositoryImpl := chartRepoRepository.NewChartRepoRepositoryImpl(db) - utilMergeUtil := util.MergeUtil{ - Logger: sugaredLogger, - } configMapServiceImpl := pipeline.NewConfigMapServiceImpl(chartRepositoryImpl, sugaredLogger, chartRepoRepositoryImpl, utilMergeUtil, pipelineConfigRepositoryImpl, configMapRepositoryImpl, envConfigOverrideRepositoryImpl, commonServiceImpl, appRepositoryImpl, configMapHistoryServiceImpl, environmentRepositoryImpl, scopedVariableCMCSManagerImpl) ciCdPipelineOrchestratorImpl := pipeline.NewCiCdPipelineOrchestrator(appRepositoryImpl, sugaredLogger, materialRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciCdConfig, appWorkflowRepositoryImpl, environmentRepositoryImpl, attributesServiceImpl, appListingRepositoryImpl, appCrudOperationServiceImpl, userAuthServiceImpl, prePostCdScriptHistoryServiceImpl, prePostCiScriptHistoryServiceImpl, pipelineStageServiceImpl, ciTemplateOverrideRepositoryImpl, gitMaterialHistoryServiceImpl, ciPipelineHistoryServiceImpl, ciTemplateServiceImpl, dockerArtifactStoreRepositoryImpl, ciArtifactRepositoryImpl, configMapServiceImpl, customTagServiceImpl, genericNoteServiceImpl) ecrConfig, err := pipeline.GetEcrConfig() From c151a32daa30fd71757f521c83e53eb79909b1a0 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 16 Jan 2024 18:30:39 +0530 Subject: [PATCH 15/64] moved validation method - 1 --- .../app/DeploymentPipelineRestHandler.go | 10 +- .../app/PipelineConfigRestHandler.go | 132 +++++++++--------- pkg/chart/ChartService.go | 87 ------------ .../DeploymentTemplateValidationService.go | 97 ++++++++++++- .../manifest/wire_deployment_manifest.go | 2 + wire_gen.go | 4 +- 6 files changed, 171 insertions(+), 161 deletions(-) diff --git a/api/restHandler/app/DeploymentPipelineRestHandler.go b/api/restHandler/app/DeploymentPipelineRestHandler.go index 22eec8e413..61776a94e8 100644 --- a/api/restHandler/app/DeploymentPipelineRestHandler.go +++ b/api/restHandler/app/DeploymentPipelineRestHandler.go @@ -119,7 +119,7 @@ func (handler PipelineConfigRestHandlerImpl) ConfigureDeploymentTemplateForApp(w scope := resourceQualifiers.Scope{ AppId: templateRequest.AppId, } - validate, err2 := handler.chartService.DeploymentTemplateValidate(r.Context(), templateRequest.ValuesOverride, chartRefId, scope) + validate, err2 := handler.deploymentTemplateValidationService.DeploymentTemplateValidate(r.Context(), templateRequest.ValuesOverride, chartRefId, scope) if !validate { common.WriteJsonResp(w, err2, nil, http.StatusBadRequest) return @@ -593,7 +593,7 @@ func (handler PipelineConfigRestHandlerImpl) ChangeChartRef(w http.ResponseWrite EnvId: request.EnvId, ClusterId: envConfigProperties.ClusterId, } - validate, err2 := handler.chartService.DeploymentTemplateValidate(r.Context(), envConfigProperties.EnvOverrideValues, envConfigProperties.ChartRefId, scope) + validate, err2 := handler.deploymentTemplateValidationService.DeploymentTemplateValidate(r.Context(), envConfigProperties.EnvOverrideValues, envConfigProperties.ChartRefId, scope) if !validate { handler.Logger.Errorw("validation err, UpdateAppOverride", "err", err2, "payload", request) common.WriteJsonResp(w, err2, "validation err, UpdateAppOverrid", http.StatusBadRequest) @@ -714,7 +714,7 @@ func (handler PipelineConfigRestHandlerImpl) EnvConfigOverrideCreate(w http.Resp EnvId: environmentId, ClusterId: envConfigProperties.ClusterId, } - validate, err2 := handler.chartService.DeploymentTemplateValidate(r.Context(), envConfigProperties.EnvOverrideValues, chartRefId, scope) + validate, err2 := handler.deploymentTemplateValidationService.DeploymentTemplateValidate(r.Context(), envConfigProperties.EnvOverrideValues, chartRefId, scope) if !validate { handler.Logger.Errorw("validation err, UpdateAppOverride", "err", err2, "payload", envConfigProperties) common.WriteJsonResp(w, err2, nil, http.StatusBadRequest) @@ -823,7 +823,7 @@ func (handler PipelineConfigRestHandlerImpl) EnvConfigOverrideUpdate(w http.Resp EnvId: envId, ClusterId: envConfigProperties.ClusterId, } - validate, err2 := handler.chartService.DeploymentTemplateValidate(r.Context(), envConfigProperties.EnvOverrideValues, chartRefId, scope) + validate, err2 := handler.deploymentTemplateValidationService.DeploymentTemplateValidate(r.Context(), envConfigProperties.EnvOverrideValues, chartRefId, scope) if !validate { handler.Logger.Errorw("validation err, UpdateAppOverride", "err", err2, "payload", envConfigProperties) common.WriteJsonResp(w, err2, nil, http.StatusBadRequest) @@ -1418,7 +1418,7 @@ func (handler PipelineConfigRestHandlerImpl) UpdateAppOverride(w http.ResponseWr AppId: templateRequest.AppId, } _, span = otel.Tracer("orchestrator").Start(ctx, "chartService.DeploymentTemplateValidate") - validate, err2 := handler.chartService.DeploymentTemplateValidate(ctx, templateRequest.ValuesOverride, chartRefId, scope) + validate, err2 := handler.deploymentTemplateValidationService.DeploymentTemplateValidate(ctx, templateRequest.ValuesOverride, chartRefId, scope) span.End() if !validate { handler.Logger.Errorw("validation err, UpdateAppOverride", "err", err2, "payload", templateRequest) diff --git a/api/restHandler/app/PipelineConfigRestHandler.go b/api/restHandler/app/PipelineConfigRestHandler.go index d58c9464d5..6a1aefe620 100644 --- a/api/restHandler/app/PipelineConfigRestHandler.go +++ b/api/restHandler/app/PipelineConfigRestHandler.go @@ -23,6 +23,7 @@ import ( "encoding/json" "fmt" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" "io" "net/http" @@ -102,41 +103,43 @@ type PipelineConfigRestHandler interface { } type PipelineConfigRestHandlerImpl struct { - pipelineBuilder pipeline.PipelineBuilder - ciPipelineRepository pipelineConfig.CiPipelineRepository - ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository - ciHandler pipeline.CiHandler - Logger *zap.SugaredLogger - chartService chart.ChartService - propertiesConfigService pipeline.PropertiesConfigService - dbMigrationService pipeline.DbMigrationService - userAuthService user.UserService - validator *validator.Validate - teamService team.TeamService - enforcer casbin.Enforcer - gitSensorClient gitSensor.Client - pipelineRepository pipelineConfig.PipelineRepository - appWorkflowService appWorkflow.AppWorkflowService - enforcerUtil rbac.EnforcerUtil - envService request.EnvironmentService - gitRegistryConfig pipeline.GitRegistryConfig - dockerRegistryConfig pipeline.DockerRegistryConfig - cdHandler pipeline.CdHandler - appCloneService appClone.AppCloneService - materialRepository pipelineConfig.MaterialRepository - policyService security2.PolicyService - scanResultRepository security.ImageScanResultRepository - gitProviderRepo repository.GitProviderRepository - argoUserService argo.ArgoUserService - imageTaggingService pipeline.ImageTaggingService - deploymentTemplateService generateManifest.DeploymentTemplateService - pipelineRestHandlerEnvConfig *PipelineRestHandlerEnvConfig - ciArtifactRepository repository.CiArtifactRepository - deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService - chartRefService chartRef.ChartRefService + pipelineBuilder pipeline.PipelineBuilder + ciPipelineRepository pipelineConfig.CiPipelineRepository + ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository + ciHandler pipeline.CiHandler + Logger *zap.SugaredLogger + deploymentTemplateValidationService deploymentTemplate.DeploymentTemplateValidationService + chartService chart.ChartService + propertiesConfigService pipeline.PropertiesConfigService + dbMigrationService pipeline.DbMigrationService + userAuthService user.UserService + validator *validator.Validate + teamService team.TeamService + enforcer casbin.Enforcer + gitSensorClient gitSensor.Client + pipelineRepository pipelineConfig.PipelineRepository + appWorkflowService appWorkflow.AppWorkflowService + enforcerUtil rbac.EnforcerUtil + envService request.EnvironmentService + gitRegistryConfig pipeline.GitRegistryConfig + dockerRegistryConfig pipeline.DockerRegistryConfig + cdHandler pipeline.CdHandler + appCloneService appClone.AppCloneService + materialRepository pipelineConfig.MaterialRepository + policyService security2.PolicyService + scanResultRepository security.ImageScanResultRepository + gitProviderRepo repository.GitProviderRepository + argoUserService argo.ArgoUserService + imageTaggingService pipeline.ImageTaggingService + deploymentTemplateService generateManifest.DeploymentTemplateService + pipelineRestHandlerEnvConfig *PipelineRestHandlerEnvConfig + ciArtifactRepository repository.CiArtifactRepository + deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService + chartRefService chartRef.ChartRefService } func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger *zap.SugaredLogger, + deploymentTemplateValidationService deploymentTemplate.DeploymentTemplateValidationService, chartService chart.ChartService, propertiesConfigService pipeline.PropertiesConfigService, dbMigrationService pipeline.DbMigrationService, @@ -166,38 +169,39 @@ func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger Logger.Errorw("error in parsing PipelineRestHandlerEnvConfig", "err", err) } return &PipelineConfigRestHandlerImpl{ - pipelineBuilder: pipelineBuilder, - Logger: Logger, - chartService: chartService, - propertiesConfigService: propertiesConfigService, - dbMigrationService: dbMigrationService, - userAuthService: userAuthService, - validator: validator, - teamService: teamService, - enforcer: enforcer, - ciHandler: ciHandler, - gitSensorClient: gitSensorClient, - ciPipelineRepository: ciPipelineRepository, - pipelineRepository: pipelineRepository, - enforcerUtil: enforcerUtil, - envService: envService, - gitRegistryConfig: gitRegistryConfig, - dockerRegistryConfig: dockerRegistryConfig, - cdHandler: cdHandler, - appCloneService: appCloneService, - appWorkflowService: appWorkflowService, - materialRepository: materialRepository, - policyService: policyService, - scanResultRepository: scanResultRepository, - gitProviderRepo: gitProviderRepo, - argoUserService: argoUserService, - ciPipelineMaterialRepository: ciPipelineMaterialRepository, - imageTaggingService: imageTaggingService, - deploymentTemplateService: deploymentTemplateService, - pipelineRestHandlerEnvConfig: envConfig, - ciArtifactRepository: ciArtifactRepository, - deployedAppMetricsService: deployedAppMetricsService, - chartRefService: chartRefService, + pipelineBuilder: pipelineBuilder, + Logger: Logger, + deploymentTemplateValidationService: deploymentTemplateValidationService, + chartService: chartService, + propertiesConfigService: propertiesConfigService, + dbMigrationService: dbMigrationService, + userAuthService: userAuthService, + validator: validator, + teamService: teamService, + enforcer: enforcer, + ciHandler: ciHandler, + gitSensorClient: gitSensorClient, + ciPipelineRepository: ciPipelineRepository, + pipelineRepository: pipelineRepository, + enforcerUtil: enforcerUtil, + envService: envService, + gitRegistryConfig: gitRegistryConfig, + dockerRegistryConfig: dockerRegistryConfig, + cdHandler: cdHandler, + appCloneService: appCloneService, + appWorkflowService: appWorkflowService, + materialRepository: materialRepository, + policyService: policyService, + scanResultRepository: scanResultRepository, + gitProviderRepo: gitProviderRepo, + argoUserService: argoUserService, + ciPipelineMaterialRepository: ciPipelineMaterialRepository, + imageTaggingService: imageTaggingService, + deploymentTemplateService: deploymentTemplateService, + pipelineRestHandlerEnvConfig: envConfig, + ciArtifactRepository: ciArtifactRepository, + deployedAppMetricsService: deployedAppMetricsService, + chartRefService: chartRefService, } } diff --git a/pkg/chart/ChartService.go b/pkg/chart/ChartService.go index 3e32659efb..f1dcf6c1dd 100644 --- a/pkg/chart/ChartService.go +++ b/pkg/chart/ChartService.go @@ -26,9 +26,7 @@ import ( "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/bean" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" bean2 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/bean" - "github.com/devtron-labs/devtron/pkg/resourceQualifiers" "github.com/devtron-labs/devtron/pkg/variables" - "github.com/devtron-labs/devtron/pkg/variables/parsers" repository5 "github.com/devtron-labs/devtron/pkg/variables/repository" "go.opentelemetry.io/otel" @@ -49,10 +47,8 @@ import ( "github.com/devtron-labs/devtron/internal/util" repository4 "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/sql" - util2 "github.com/devtron-labs/devtron/util" "github.com/go-pg/pg" "github.com/juju/errors" - "github.com/xeipuuv/gojsonschema" "go.uber.org/zap" "k8s.io/helm/pkg/proto/hapi/chart" "sigs.k8s.io/yaml" @@ -67,7 +63,6 @@ type ChartService interface { IsReadyToTrigger(appId int, envId int, pipelineId int) (IsReady, error) FindPreviousChartByAppId(appId int) (chartTemplate *TemplateRequest, err error) UpgradeForApp(appId int, chartRefId int, newAppOverride map[string]interface{}, userId int32, ctx context.Context) (bool, error) - DeploymentTemplateValidate(ctx context.Context, templatejson interface{}, chartRefId int, scope resourceQualifiers.Scope) (bool, error) CheckIfChartRefUserUploadedByAppId(id int) (bool, error) PatchEnvOverrides(values json.RawMessage, oldChartType string, newChartType string) (json.RawMessage, error) FlaggerCanaryEnabled(values json.RawMessage) (bool, error) @@ -859,88 +854,6 @@ func (impl ChartServiceImpl) UpgradeForApp(appId int, chartRefId int, newAppOver return true, nil } -const memoryPattern = `"1000Mi" or "1Gi"` -const cpuPattern = `"50m" or "0.05"` -const cpu = "cpu" -const memory = "memory" - -func (impl ChartServiceImpl) DeploymentTemplateValidate(ctx context.Context, template interface{}, chartRefId int, scope resourceQualifiers.Scope) (bool, error) { - _, span := otel.Tracer("orchestrator").Start(ctx, "JsonSchemaExtractFromFile") - schemajson, version, err := impl.chartRefService.JsonSchemaExtractFromFile(chartRefId) - span.End() - if err != nil { - impl.logger.Errorw("Json Schema not found err, FindJsonSchema", "err", err) - return true, nil - } - //if err != nil && chartRefId >= 9 { - // impl.logger.Errorw("Json Schema not found err, FindJsonSchema", "err", err) - // return false, err - //} else if err != nil { - // impl.logger.Errorw("Json Schema not found err, FindJsonSchema", "err", err) - // return true, nil - //} - - templateBytes := template.(json.RawMessage) - templatejsonstring, _, err := impl.scopedVariableManager.ExtractVariablesAndResolveTemplate(scope, string(templateBytes), parsers.JsonVariableTemplate, true, false) - if err != nil { - return false, err - } - var templatejson interface{} - err = json.Unmarshal([]byte(templatejsonstring), &templatejson) - if err != nil { - fmt.Println("Error:", err) - return false, err - } - - schemaLoader := gojsonschema.NewGoLoader(schemajson) - documentLoader := gojsonschema.NewGoLoader(templatejson) - marshalTemplatejson, err := json.Marshal(templatejson) - if err != nil { - impl.logger.Errorw("json template marshal err, DeploymentTemplateValidate", "err", err) - return false, err - } - _, span = otel.Tracer("orchestrator").Start(ctx, "gojsonschema.Validate") - result, err := gojsonschema.Validate(schemaLoader, documentLoader) - span.End() - if err != nil { - impl.logger.Errorw("result validate err, DeploymentTemplateValidate", "err", err) - return false, err - } - if result.Valid() { - var dat map[string]interface{} - if err := json.Unmarshal(marshalTemplatejson, &dat); err != nil { - impl.logger.Errorw("json template unmarshal err, DeploymentTemplateValidate", "err", err) - return false, err - } - - _, err := util2.CompareLimitsRequests(dat, version) - if err != nil { - impl.logger.Errorw("LimitRequestCompare err, DeploymentTemplateValidate", "err", err) - return false, err - } - _, err = util2.AutoScale(dat) - if err != nil { - impl.logger.Errorw("LimitRequestCompare err, DeploymentTemplateValidate", "err", err) - return false, err - } - - return true, nil - } else { - var stringerror string - for _, err := range result.Errors() { - impl.logger.Errorw("result err, DeploymentTemplateValidate", "err", err.Details()) - if err.Details()["format"] == cpu { - stringerror = stringerror + err.Field() + ": Format should be like " + cpuPattern + "\n" - } else if err.Details()["format"] == memory { - stringerror = stringerror + err.Field() + ": Format should be like " + memoryPattern + "\n" - } else { - stringerror = stringerror + err.String() + "\n" - } - } - return false, errors.New(stringerror) - } -} - func (impl ChartServiceImpl) CheckIfChartRefUserUploadedByAppId(id int) (bool, error) { chartInfo, err := impl.chartRepository.FindLatestChartForAppByAppId(id) if err != nil { diff --git a/pkg/deployment/manifest/deploymentTemplate/DeploymentTemplateValidationService.go b/pkg/deployment/manifest/deploymentTemplate/DeploymentTemplateValidationService.go index 6c851bc518..25023f1671 100644 --- a/pkg/deployment/manifest/deploymentTemplate/DeploymentTemplateValidationService.go +++ b/pkg/deployment/manifest/deploymentTemplate/DeploymentTemplateValidationService.go @@ -1,16 +1,105 @@ package deploymentTemplate -import "go.uber.org/zap" +import ( + "context" + "encoding/json" + "errors" + "fmt" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/bean" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" + "github.com/devtron-labs/devtron/pkg/resourceQualifiers" + "github.com/devtron-labs/devtron/pkg/variables" + "github.com/devtron-labs/devtron/pkg/variables/parsers" + util2 "github.com/devtron-labs/devtron/util" + "github.com/xeipuuv/gojsonschema" + "go.opentelemetry.io/otel" + "go.uber.org/zap" +) type DeploymentTemplateValidationService interface { + DeploymentTemplateValidate(ctx context.Context, template interface{}, chartRefId int, scope resourceQualifiers.Scope) (bool, error) } type DeploymentTemplateValidationServiceImpl struct { - logger *zap.SugaredLogger + logger *zap.SugaredLogger + chartRefService chartRef.ChartRefService + scopedVariableManager variables.ScopedVariableManager } -func NewDeploymentTemplateValidationServiceImpl(logger *zap.SugaredLogger) *DeploymentTemplateValidationServiceImpl { +func NewDeploymentTemplateValidationServiceImpl(logger *zap.SugaredLogger, + chartRefService chartRef.ChartRefService, + scopedVariableManager variables.ScopedVariableManager) *DeploymentTemplateValidationServiceImpl { return &DeploymentTemplateValidationServiceImpl{ - logger: logger, + logger: logger, + chartRefService: chartRefService, + scopedVariableManager: scopedVariableManager, + } +} + +func (impl *DeploymentTemplateValidationServiceImpl) DeploymentTemplateValidate(ctx context.Context, template interface{}, chartRefId int, scope resourceQualifiers.Scope) (bool, error) { + _, span := otel.Tracer("orchestrator").Start(ctx, "JsonSchemaExtractFromFile") + schemajson, version, err := impl.chartRefService.JsonSchemaExtractFromFile(chartRefId) + span.End() + if err != nil { + impl.logger.Errorw("Json Schema not found err, FindJsonSchema", "err", err) + return true, nil + } + + templateBytes := template.(json.RawMessage) + templatejsonstring, _, err := impl.scopedVariableManager.ExtractVariablesAndResolveTemplate(scope, string(templateBytes), parsers.JsonVariableTemplate, true, false) + if err != nil { + return false, err + } + var templatejson interface{} + err = json.Unmarshal([]byte(templatejsonstring), &templatejson) + if err != nil { + fmt.Println("Error:", err) + return false, err + } + + schemaLoader := gojsonschema.NewGoLoader(schemajson) + documentLoader := gojsonschema.NewGoLoader(templatejson) + marshalTemplatejson, err := json.Marshal(templatejson) + if err != nil { + impl.logger.Errorw("json template marshal err, DeploymentTemplateValidate", "err", err) + return false, err + } + _, span = otel.Tracer("orchestrator").Start(ctx, "gojsonschema.Validate") + result, err := gojsonschema.Validate(schemaLoader, documentLoader) + span.End() + if err != nil { + impl.logger.Errorw("result validate err, DeploymentTemplateValidate", "err", err) + return false, err + } + if result.Valid() { + var dat map[string]interface{} + if err := json.Unmarshal(marshalTemplatejson, &dat); err != nil { + impl.logger.Errorw("json template unmarshal err, DeploymentTemplateValidate", "err", err) + return false, err + } + _, err := util2.CompareLimitsRequests(dat, version) + if err != nil { + impl.logger.Errorw("LimitRequestCompare err, DeploymentTemplateValidate", "err", err) + return false, err + } + _, err = util2.AutoScale(dat) + if err != nil { + impl.logger.Errorw("LimitRequestCompare err, DeploymentTemplateValidate", "err", err) + return false, err + } + return true, nil + } else { + var stringerror string + for _, err := range result.Errors() { + impl.logger.Errorw("result err, DeploymentTemplateValidate", "err", err.Details()) + if err.Details()["format"] == bean.Cpu { + stringerror = stringerror + err.Field() + ": Format should be like " + bean.CpuPattern + "\n" + } else if err.Details()["format"] == bean.Memory { + stringerror = stringerror + err.Field() + ": Format should be like " + bean.MemoryPattern + "\n" + } else { + stringerror = stringerror + err.String() + "\n" + } + } + return false, errors.New(stringerror) } } diff --git a/pkg/deployment/manifest/wire_deployment_manifest.go b/pkg/deployment/manifest/wire_deployment_manifest.go index fe6d1a6e61..d1d9613d40 100644 --- a/pkg/deployment/manifest/wire_deployment_manifest.go +++ b/pkg/deployment/manifest/wire_deployment_manifest.go @@ -12,6 +12,8 @@ var DeploymentManifestWireSet = wire.NewSet( wire.Bind(new(deployedAppMetrics.DeployedAppMetricsService), new(*deployedAppMetrics.DeployedAppMetricsServiceImpl)), deploymentTemplate.NewDeploymentTemplateServiceImpl, wire.Bind(new(deploymentTemplate.DeploymentTemplateService), new(*deploymentTemplate.DeploymentTemplateServiceImpl)), + deploymentTemplate.NewDeploymentTemplateValidationServiceImpl, + wire.Bind(new(deploymentTemplate.DeploymentTemplateValidationService), new(*deploymentTemplate.DeploymentTemplateValidationServiceImpl)), chartRef.NewChartRefServiceImpl, wire.Bind(new(chartRef.ChartRefService), new(*chartRef.ChartRefServiceImpl)), ) diff --git a/wire_gen.go b/wire_gen.go index 5e774bbe7a..712f28e051 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -102,6 +102,7 @@ import ( "github.com/devtron-labs/devtron/pkg/commonService" delete2 "github.com/devtron-labs/devtron/pkg/delete" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" "github.com/devtron-labs/devtron/pkg/deploymentGroup" "github.com/devtron-labs/devtron/pkg/devtronResource" @@ -494,6 +495,7 @@ func InitializeApp() (*App, error) { appDeploymentTypeChangeManagerImpl := pipeline.NewAppDeploymentTypeChangeManagerImpl(sugaredLogger, pipelineRepositoryImpl, workflowDagExecutorImpl, appServiceImpl, chartTemplateServiceImpl, appStatusRepositoryImpl, helmAppServiceImpl, applicationServiceClientImpl, appArtifactManagerImpl, cdPipelineConfigServiceImpl) devtronAppConfigServiceImpl := pipeline.NewDevtronAppConfigServiceImpl(sugaredLogger, ciCdPipelineOrchestratorImpl, appRepositoryImpl, pipelineRepositoryImpl, resourceGroupServiceImpl, enforcerUtilImpl, ciMaterialConfigServiceImpl) pipelineBuilderImpl := pipeline.NewPipelineBuilderImpl(sugaredLogger, materialRepositoryImpl, chartRepositoryImpl, ciPipelineConfigServiceImpl, ciMaterialConfigServiceImpl, appArtifactManagerImpl, devtronAppCMCSServiceImpl, devtronAppStrategyServiceImpl, appDeploymentTypeChangeManagerImpl, cdPipelineConfigServiceImpl, devtronAppConfigServiceImpl) + deploymentTemplateValidationServiceImpl := deploymentTemplate.NewDeploymentTemplateValidationServiceImpl(sugaredLogger, chartRefServiceImpl, scopedVariableManagerImpl) chartServiceImpl := chart.NewChartServiceImpl(chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, appRepositoryImpl, utilMergeUtil, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, environmentRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) dbMigrationServiceImpl := pipeline.NewDbMogrationService(sugaredLogger, dbMigrationConfigRepositoryImpl) ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciWorkflowRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, mergeUtil, ciPipelineRepositoryImpl, prePostCiScriptHistoryServiceImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateServiceImpl, appCrudOperationServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, customTagServiceImpl, pluginInputVariableParserImpl, globalPluginServiceImpl) @@ -516,7 +518,7 @@ func InitializeApp() (*App, error) { imageScanObjectMetaRepositoryImpl := security.NewImageScanObjectMetaRepositoryImpl(db, sugaredLogger) cveStoreRepositoryImpl := security.NewCveStoreRepositoryImpl(db, sugaredLogger) policyServiceImpl := security2.NewPolicyServiceImpl(environmentServiceImpl, sugaredLogger, appRepositoryImpl, pipelineOverrideRepositoryImpl, cvePolicyRepositoryImpl, clusterServiceImplExtended, pipelineRepositoryImpl, imageScanResultRepositoryImpl, imageScanDeployInfoRepositoryImpl, imageScanObjectMetaRepositoryImpl, httpClient, ciArtifactRepositoryImpl, ciCdConfig, imageScanHistoryRepositoryImpl, cveStoreRepositoryImpl, ciTemplateRepositoryImpl) - pipelineConfigRestHandlerImpl := app3.NewPipelineRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, chartServiceImpl, propertiesConfigServiceImpl, dbMigrationServiceImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, environmentServiceImpl, gitRegistryConfigImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, deploymentTemplateServiceImpl, appWorkflowServiceImpl, materialRepositoryImpl, policyServiceImpl, imageScanResultRepositoryImpl, gitProviderRepositoryImpl, argoUserServiceImpl, ciPipelineMaterialRepositoryImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) + pipelineConfigRestHandlerImpl := app3.NewPipelineRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, deploymentTemplateValidationServiceImpl, chartServiceImpl, propertiesConfigServiceImpl, dbMigrationServiceImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, environmentServiceImpl, gitRegistryConfigImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, deploymentTemplateServiceImpl, appWorkflowServiceImpl, materialRepositoryImpl, policyServiceImpl, imageScanResultRepositoryImpl, gitProviderRepositoryImpl, argoUserServiceImpl, ciPipelineMaterialRepositoryImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) appWorkflowRestHandlerImpl := restHandler.NewAppWorkflowRestHandlerImpl(sugaredLogger, userServiceImpl, appWorkflowServiceImpl, teamServiceImpl, enforcerImpl, pipelineBuilderImpl, appRepositoryImpl, enforcerUtilImpl) webhookEventDataRepositoryImpl := repository.NewWebhookEventDataRepositoryImpl(db) webhookEventDataConfigImpl := pipeline.NewWebhookEventDataConfigImpl(sugaredLogger, webhookEventDataRepositoryImpl) From 20562f21809e7574399fc8a83303e8dd3800c3a1 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 16 Jan 2024 18:44:48 +0530 Subject: [PATCH 16/64] wip --- api/restHandler/CoreAppRestHandler.go | 4 +--- .../app/DeploymentPipelineRestHandler.go | 2 +- pkg/chart/ChartService.go | 21 ------------------ .../DeploymentTemplateValidationService.go | 22 +++++++++++++++++++ .../deploymentTemplate/adapter/adapter.go | 1 + .../manifest/deploymentTemplate/bean/bean.go | 11 ++++++++++ 6 files changed, 36 insertions(+), 25 deletions(-) create mode 100644 pkg/deployment/manifest/deploymentTemplate/adapter/adapter.go create mode 100644 pkg/deployment/manifest/deploymentTemplate/bean/bean.go diff --git a/api/restHandler/CoreAppRestHandler.go b/api/restHandler/CoreAppRestHandler.go index 5a0ccf43f4..3264af0409 100644 --- a/api/restHandler/CoreAppRestHandler.go +++ b/api/restHandler/CoreAppRestHandler.go @@ -93,7 +93,6 @@ type CoreAppRestHandlerImpl struct { appWorkflowRepository appWorkflow2.AppWorkflowRepository environmentRepository repository2.EnvironmentRepository configMapRepository chartConfig.ConfigMapRepository - envConfigRepo chartConfig.EnvConfigOverrideRepository chartRepo chartRepoRepository.ChartRepository teamService team.TeamService argoUserService argo.ArgoUserService @@ -107,7 +106,7 @@ func NewCoreAppRestHandlerImpl(logger *zap.SugaredLogger, userAuthService user.U propertiesConfigService pipeline.PropertiesConfigService, appWorkflowService appWorkflow.AppWorkflowService, materialRepository pipelineConfig.MaterialRepository, gitProviderRepo repository.GitProviderRepository, appWorkflowRepository appWorkflow2.AppWorkflowRepository, environmentRepository repository2.EnvironmentRepository, configMapRepository chartConfig.ConfigMapRepository, - envConfigRepo chartConfig.EnvConfigOverrideRepository, chartRepo chartRepoRepository.ChartRepository, teamService team.TeamService, + chartRepo chartRepoRepository.ChartRepository, teamService team.TeamService, argoUserService argo.ArgoUserService, pipelineStageService pipeline.PipelineStageService, ciPipelineRepository pipelineConfig.CiPipelineRepository) *CoreAppRestHandlerImpl { handler := &CoreAppRestHandlerImpl{ logger: logger, @@ -128,7 +127,6 @@ func NewCoreAppRestHandlerImpl(logger *zap.SugaredLogger, userAuthService user.U appWorkflowRepository: appWorkflowRepository, environmentRepository: environmentRepository, configMapRepository: configMapRepository, - envConfigRepo: envConfigRepo, chartRepo: chartRepo, teamService: teamService, argoUserService: argoUserService, diff --git a/api/restHandler/app/DeploymentPipelineRestHandler.go b/api/restHandler/app/DeploymentPipelineRestHandler.go index 61776a94e8..c887889a61 100644 --- a/api/restHandler/app/DeploymentPipelineRestHandler.go +++ b/api/restHandler/app/DeploymentPipelineRestHandler.go @@ -555,7 +555,7 @@ func (handler PipelineConfigRestHandlerImpl) ChangeChartRef(w http.ResponseWrite } if newChartType == bean4.RolloutChartType { - enabled, err := handler.chartService.FlaggerCanaryEnabled(envConfigProperties.EnvOverrideValues) + enabled, err := handler.deploymentTemplateValidationService.FlaggerCanaryEnabled(envConfigProperties.EnvOverrideValues) if err != nil || enabled { handler.Logger.Errorw("rollout charts do not support flaggerCanary, ChangeChartRef", "err", err, "payload", request) common.WriteJsonResp(w, err, "rollout charts do not support flaggerCanary, ChangeChartRef", http.StatusBadRequest) diff --git a/pkg/chart/ChartService.go b/pkg/chart/ChartService.go index f1dcf6c1dd..d6acd1c587 100644 --- a/pkg/chart/ChartService.go +++ b/pkg/chart/ChartService.go @@ -65,7 +65,6 @@ type ChartService interface { UpgradeForApp(appId int, chartRefId int, newAppOverride map[string]interface{}, userId int32, ctx context.Context) (bool, error) CheckIfChartRefUserUploadedByAppId(id int) (bool, error) PatchEnvOverrides(values json.RawMessage, oldChartType string, newChartType string) (json.RawMessage, error) - FlaggerCanaryEnabled(values json.RawMessage) (bool, error) ChartRefAutocompleteForAppOrEnv(appId int, envId int) (*bean2.ChartRefAutocompleteResponse, error) } @@ -116,26 +115,6 @@ func NewChartServiceImpl(chartRepository chartRepoRepository.ChartRepository, } } -func (impl ChartServiceImpl) FlaggerCanaryEnabled(values json.RawMessage) (bool, error) { - var jsonMap map[string]json.RawMessage - if err := json.Unmarshal([]byte(values), &jsonMap); err != nil { - return false, err - } - - flaggerCanary, found := jsonMap["flaggerCanary"] - if !found { - return false, nil - } - var flaggerCanaryUnmarshalled map[string]json.RawMessage - if err := json.Unmarshal([]byte(flaggerCanary), &flaggerCanaryUnmarshalled); err != nil { - return false, err - } - enabled, found := flaggerCanaryUnmarshalled["enabled"] - if !found { - return true, fmt.Errorf("flagger canary enabled field must be set and be equal to false") - } - return string(enabled) == "true", nil -} func (impl ChartServiceImpl) PatchEnvOverrides(values json.RawMessage, oldChartType string, newChartType string) (json.RawMessage, error) { return PatchWinterSoldierConfig(values, newChartType) } diff --git a/pkg/deployment/manifest/deploymentTemplate/DeploymentTemplateValidationService.go b/pkg/deployment/manifest/deploymentTemplate/DeploymentTemplateValidationService.go index 25023f1671..8c0bf25b06 100644 --- a/pkg/deployment/manifest/deploymentTemplate/DeploymentTemplateValidationService.go +++ b/pkg/deployment/manifest/deploymentTemplate/DeploymentTemplateValidationService.go @@ -18,6 +18,7 @@ import ( type DeploymentTemplateValidationService interface { DeploymentTemplateValidate(ctx context.Context, template interface{}, chartRefId int, scope resourceQualifiers.Scope) (bool, error) + FlaggerCanaryEnabled(values json.RawMessage) (bool, error) } type DeploymentTemplateValidationServiceImpl struct { @@ -103,3 +104,24 @@ func (impl *DeploymentTemplateValidationServiceImpl) DeploymentTemplateValidate( return false, errors.New(stringerror) } } + +func (impl *DeploymentTemplateValidationServiceImpl) FlaggerCanaryEnabled(values json.RawMessage) (bool, error) { + var jsonMap map[string]json.RawMessage + if err := json.Unmarshal(values, &jsonMap); err != nil { + return false, err + } + + flaggerCanary, found := jsonMap[bean.FlaggerCanary] + if !found { + return false, nil + } + var flaggerCanaryUnmarshalled map[string]json.RawMessage + if err := json.Unmarshal(flaggerCanary, &flaggerCanaryUnmarshalled); err != nil { + return false, err + } + enabled, found := flaggerCanaryUnmarshalled[bean.EnabledFlag] + if !found { + return true, fmt.Errorf("flagger canary enabled field must be set and be equal to false") + } + return string(enabled) == bean.TrueFlag, nil +} diff --git a/pkg/deployment/manifest/deploymentTemplate/adapter/adapter.go b/pkg/deployment/manifest/deploymentTemplate/adapter/adapter.go new file mode 100644 index 0000000000..b8e8da3083 --- /dev/null +++ b/pkg/deployment/manifest/deploymentTemplate/adapter/adapter.go @@ -0,0 +1 @@ +package adapter diff --git a/pkg/deployment/manifest/deploymentTemplate/bean/bean.go b/pkg/deployment/manifest/deploymentTemplate/bean/bean.go new file mode 100644 index 0000000000..78671118fc --- /dev/null +++ b/pkg/deployment/manifest/deploymentTemplate/bean/bean.go @@ -0,0 +1,11 @@ +package bean + +const ( + MemoryPattern = `"1000Mi" or "1Gi"` + CpuPattern = `"50m" or "0.05"` + Cpu = "cpu" + Memory = "memory" + FlaggerCanary = "flaggerCanary" + EnabledFlag = "enabled" + TrueFlag = "true" +) From 75b99abca19ad0a3ad3361ce80e9334213ad6df7 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 16 Jan 2024 18:45:43 +0530 Subject: [PATCH 17/64] removed redundant appMetrics req obj --- pkg/chart/bean.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/pkg/chart/bean.go b/pkg/chart/bean.go index c3dce0a3b1..eb9599d922 100644 --- a/pkg/chart/bean.go +++ b/pkg/chart/bean.go @@ -35,13 +35,6 @@ type TemplateRequest struct { UserId int32 `json:"-"` } -type AppMetricEnableDisableRequest struct { - AppId int `json:"appId,omitempty"` - EnvironmentId int `json:"environmentId,omitempty"` - IsAppMetricsEnabled bool `json:"isAppMetricsEnabled"` - UserId int32 `json:"-"` -} - type ChartUpgradeRequest struct { ChartRefId int `json:"chartRefId" validate:"number"` All bool `json:"all"` From 11f76ff7d45cf2cbfa93a67368988311346802e3 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 16 Jan 2024 19:03:26 +0530 Subject: [PATCH 18/64] moved app metrics bindings to wireset --- Wire.go | 6 ------ pkg/deployment/manifest/wire_deployment_manifest.go | 6 ++++++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Wire.go b/Wire.go index 79e4c8a53a..0dc8fefc60 100644 --- a/Wire.go +++ b/Wire.go @@ -97,7 +97,6 @@ import ( "github.com/devtron-labs/devtron/pkg/commonService" delete2 "github.com/devtron-labs/devtron/pkg/delete" "github.com/devtron-labs/devtron/pkg/deployment/manifest" - repository11 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/repository" "github.com/devtron-labs/devtron/pkg/deploymentGroup" "github.com/devtron-labs/devtron/pkg/devtronResource" repository9 "github.com/devtron-labs/devtron/pkg/devtronResource/repository" @@ -552,11 +551,6 @@ func InitializeApp() (*App, error) { restHandler.NewExternalCiRestHandlerImpl, wire.Bind(new(restHandler.ExternalCiRestHandler), new(*restHandler.ExternalCiRestHandlerImpl)), - repository11.NewAppLevelMetricsRepositoryImpl, - wire.Bind(new(repository11.AppLevelMetricsRepository), new(*repository11.AppLevelMetricsRepositoryImpl)), - - repository11.NewEnvLevelAppMetricsRepositoryImpl, - wire.Bind(new(repository11.EnvLevelAppMetricsRepository), new(*repository11.EnvLevelAppMetricsRepositoryImpl)), grafana.GetGrafanaClientConfig, grafana.NewGrafanaClientImpl, diff --git a/pkg/deployment/manifest/wire_deployment_manifest.go b/pkg/deployment/manifest/wire_deployment_manifest.go index fe6d1a6e61..497771f975 100644 --- a/pkg/deployment/manifest/wire_deployment_manifest.go +++ b/pkg/deployment/manifest/wire_deployment_manifest.go @@ -2,12 +2,18 @@ package manifest import ( "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/repository" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" "github.com/google/wire" ) var DeploymentManifestWireSet = wire.NewSet( + repository.NewAppLevelMetricsRepositoryImpl, + wire.Bind(new(repository.AppLevelMetricsRepository), new(*repository.AppLevelMetricsRepositoryImpl)), + repository.NewEnvLevelAppMetricsRepositoryImpl, + wire.Bind(new(repository.EnvLevelAppMetricsRepository), new(*repository.EnvLevelAppMetricsRepositoryImpl)), + deployedAppMetrics.NewDeployedAppMetricsServiceImpl, wire.Bind(new(deployedAppMetrics.DeployedAppMetricsService), new(*deployedAppMetrics.DeployedAppMetricsServiceImpl)), deploymentTemplate.NewDeploymentTemplateServiceImpl, From b59cb1cf8daa7dfb0b042798b6bfb1003c5eab09 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 16 Jan 2024 19:15:48 +0530 Subject: [PATCH 19/64] removed multiple dead code --- api/restHandler/BatchOperationRestHandler.go | 12 ------- api/restHandler/CoreAppRestHandler.go | 14 -------- .../app/PipelineConfigRestHandler.go | 29 ----------------- api/restHandler/common/apiError.go | 32 ------------------- .../sql/repository/AppListingRepository.go | 25 --------------- pkg/chart/ChartService.go | 25 --------------- pkg/pipeline/PropertiesConfig.go | 13 -------- 7 files changed, 150 deletions(-) diff --git a/api/restHandler/BatchOperationRestHandler.go b/api/restHandler/BatchOperationRestHandler.go index d89bafbd55..1c1178568a 100644 --- a/api/restHandler/BatchOperationRestHandler.go +++ b/api/restHandler/BatchOperationRestHandler.go @@ -145,15 +145,3 @@ func validatePipeline(pipeline *v1.Pipeline, props v1.InheritedProps) error { } return nil } - -func executePipeline(pipeline *v1.Pipeline, props v1.InheritedProps) error { - if pipeline.Build == nil && pipeline.Deployment == nil { - return nil - } else if pipeline.Build != nil { - pipeline.Build.UpdateMissingProps(props) - return validation.ValidateBuild(pipeline.Build) - } else if pipeline.Deployment != nil { - //return batch.ExecuteDeployment(pipeline.Deployment, props) - } - return nil -} diff --git a/api/restHandler/CoreAppRestHandler.go b/api/restHandler/CoreAppRestHandler.go index 3264af0409..b8098c2f3d 100644 --- a/api/restHandler/CoreAppRestHandler.go +++ b/api/restHandler/CoreAppRestHandler.go @@ -2086,20 +2086,6 @@ func convertCdDeploymentStrategies(deploymentStrategies []*appBean.DeploymentStr return convertedStrategies, nil } -func ExtractErrorType(err error) int { - switch err.(type) { - case *util2.InternalServerError: - return http.StatusInternalServerError - case *util2.ForbiddenError: - return http.StatusForbidden - case *util2.BadRequestError: - return http.StatusBadRequest - default: - //TODO : ask and update response for this case - return 0 - } -} - func (handler CoreAppRestHandlerImpl) validateCdPipelines(cdPipelines []*appBean.CdPipelineDetails, appName, token string) (error, int) { for _, cdPipeline := range cdPipelines { envName := cdPipeline.EnvironmentName diff --git a/api/restHandler/app/PipelineConfigRestHandler.go b/api/restHandler/app/PipelineConfigRestHandler.go index 6a1aefe620..5de61b9131 100644 --- a/api/restHandler/app/PipelineConfigRestHandler.go +++ b/api/restHandler/app/PipelineConfigRestHandler.go @@ -44,7 +44,6 @@ import ( "github.com/go-pg/pg" "go.opentelemetry.io/otel" - bean2 "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/sql/repository/security" @@ -59,10 +58,7 @@ import ( util2 "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/rbac" "github.com/gorilla/mux" - "github.com/grpc-ecosystem/grpc-gateway/runtime" "go.uber.org/zap" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" "gopkg.in/go-playground/validator.v9" ) @@ -573,31 +569,6 @@ func (handler *PipelineConfigRestHandlerImpl) sendData(event []byte, w http.Resp } } -func (handler *PipelineConfigRestHandlerImpl) handleForwardResponseStreamError(wroteHeader bool, w http.ResponseWriter, err error) { - code := "000" - if !wroteHeader { - s, ok := status.FromError(err) - if !ok { - s = status.New(codes.Unknown, err.Error()) - } - w.WriteHeader(runtime.HTTPStatusFromCode(s.Code())) - code = fmt.Sprint(s.Code()) - } - response := bean2.Response{} - apiErr := bean2.ApiError{} - apiErr.Code = code // 000=unknown - apiErr.InternalMessage = err.Error() - response.Errors = []bean2.ApiError{apiErr} - buf, err2 := json.Marshal(response) - if err2 != nil { - handler.Logger.Errorw("marshal err, handleForwardResponseStreamError", "err", err2, "response", response) - } - if _, err3 := w.Write(buf); err3 != nil { - handler.Logger.Errorw("Failed to notify error to client, handleForwardResponseStreamError", "err", err3, "response", response) - return - } -} - func (handler PipelineConfigRestHandlerImpl) FetchAppWorkflowStatusForTriggerView(w http.ResponseWriter, r *http.Request) { userId, err := handler.userAuthService.GetLoggedInUser(r) if userId == 0 || err != nil { diff --git a/api/restHandler/common/apiError.go b/api/restHandler/common/apiError.go index 8d4c29689d..b95074876b 100644 --- a/api/restHandler/common/apiError.go +++ b/api/restHandler/common/apiError.go @@ -126,29 +126,6 @@ func WriteJsonResp(w http.ResponseWriter, err error, respBody interface{}, statu } -// use this method when we have specific api error to be conveyed to api User -func writeJsonRespStructured(w http.ResponseWriter, err error, respBody interface{}, status int, apiErrors []*util.ApiError) { - response := Response{} - response.Code = status - response.Status = http.StatusText(status) - if err == nil { - response.Result = respBody - } else { - response.Errors = apiErrors - } - b, err := json.Marshal(response) - if err != nil { - util.GetLogger().Error("error in marshaling err object", err) - status = 500 - } - w.Header().Set(CONTENT_TYPE, APPLICATION_JSON) - w.WriteHeader(status) - _, err = w.Write(b) - if err != nil { - util.GetLogger().Error(err) - } -} - // global response body used across api type Response struct { Code int `json:"code,omitempty"` @@ -156,12 +133,3 @@ type Response struct { Result interface{} `json:"result,omitempty"` Errors []*util.ApiError `json:"errors,omitempty"` } - -func contains(s []*string, e *string) bool { - for _, a := range s { - if a == e { - return true - } - } - return false -} diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index c5e2583b53..150ecfd37a 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -181,18 +181,6 @@ func (impl AppListingRepositoryImpl) FetchJobsLastSucceededOn(CiPipelineIDs []in return lastSucceededTimeArray, nil } -func getRequiredAppIdsInSequence(appIds []int) []int { - resIDs := make([]int, 0) - appIdsSet := make(map[int]bool) - for _, appId := range appIds { - if _, ok := appIdsSet[appId]; !ok { - resIDs = append(resIDs, appId) - appIdsSet[appId] = true - } - } - return resIDs -} - func (impl AppListingRepositoryImpl) FetchAppsByEnvironment(appListingFilter helper.AppListingFilter) ([]*bean.AppEnvironmentContainer, error) { impl.Logger.Debug("reached at FetchAppsByEnvironment:") var appEnvArr []*bean.AppEnvironmentContainer @@ -492,19 +480,6 @@ func (impl AppListingRepositoryImpl) FetchAppDetail(ctx context.Context, appId i return appDetailContainer, nil } -func (impl AppListingRepositoryImpl) fetchLinkOutsByAppIdAndEnvId(appId int, envId int) ([]string, error) { - impl.Logger.Debug("reached at AppListingRepository:") - - var linkOuts []string - query := "SELECT ael.link from app_env_linkouts ael where ael.app_id=? and ael.environment_id=?" - impl.Logger.Debugw("lingOut query:", query) - _, err := impl.dbConnection.Query(&linkOuts, query, appId, envId) - if err != nil { - impl.Logger.Errorw("err", err) - } - return linkOuts, err -} - func (impl AppListingRepositoryImpl) PrometheusApiByEnvId(id int) (*string, error) { impl.Logger.Debug("reached at PrometheusApiByEnvId:") var prometheusEndpoint string diff --git a/pkg/chart/ChartService.go b/pkg/chart/ChartService.go index 1853c45bb0..9939447811 100644 --- a/pkg/chart/ChartService.go +++ b/pkg/chart/ChartService.go @@ -119,10 +119,6 @@ func (impl ChartServiceImpl) PatchEnvOverrides(values json.RawMessage, oldChartT return PatchWinterSoldierConfig(values, newChartType) } -type AppMetricsEnabled struct { - AppMetrics bool `json:"app-metrics"` -} - func (impl ChartServiceImpl) Create(templateRequest TemplateRequest, ctx context.Context) (*TemplateRequest, error) { err := impl.chartRefService.CheckChartExists(templateRequest.ChartRefId) if err != nil { @@ -433,27 +429,6 @@ func (impl ChartServiceImpl) getChartMetaData(templateRequest TemplateRequest) ( return metadata, err } -func (impl ChartServiceImpl) getRefChartVersion(templateRequest TemplateRequest) (string, error) { - var version string - if templateRequest.ChartRefId > 0 { - chartRefDto, err := impl.chartRefService.FindById(templateRequest.ChartRefId) - if err != nil { - chartRefDto, err = impl.chartRefService.GetDefault() - if err != nil { - return "", err - } - } - version = chartRefDto.Version - } else { - chartRefDto, err := impl.chartRefService.GetDefault() - if err != nil { - return "", err - } - version = chartRefDto.Location - } - return version, nil -} - func (impl ChartServiceImpl) getChartRepo(templateRequest TemplateRequest) (*chartRepoRepository.ChartRepo, error) { if templateRequest.ChartRepositoryId == 0 { chartRepo, err := impl.repoRepository.GetDefault() diff --git a/pkg/pipeline/PropertiesConfig.go b/pkg/pipeline/PropertiesConfig.go index 3f997d2a6a..56cce5f16e 100644 --- a/pkg/pipeline/PropertiesConfig.go +++ b/pkg/pipeline/PropertiesConfig.go @@ -28,7 +28,6 @@ import ( repository5 "github.com/devtron-labs/devtron/pkg/variables/repository" "time" - chartService "github.com/devtron-labs/devtron/pkg/chart" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/pipeline/history" @@ -331,18 +330,6 @@ func (impl PropertiesConfigServiceImpl) UpdateEnvironmentProperties(appId int, p return propertiesRequest, err } -func (impl PropertiesConfigServiceImpl) buildAppMetricsJson() ([]byte, error) { - appMetricsEnabled := chartService.AppMetricsEnabled{ - AppMetrics: true, - } - appMetricsJson, err := json.Marshal(appMetricsEnabled) - if err != nil { - impl.logger.Error(err) - return nil, err - } - return appMetricsJson, nil -} - func (impl PropertiesConfigServiceImpl) CreateIfRequired(chart *chartRepoRepository.Chart, environmentId int, userId int32, manualReviewed bool, chartStatus models.ChartStatus, isOverride, isAppMetricsEnabled bool, namespace string, IsBasicViewLocked bool, CurrentViewEditor models.ChartsViewEditorType, tx *pg.Tx) (*chartConfig.EnvConfigOverride, bool, error) { env, err := impl.environmentRepository.FindById(environmentId) if err != nil { From d113b35b51d021aa2d461d8cbbd22318cc15cd0d Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 16 Jan 2024 19:34:48 +0530 Subject: [PATCH 20/64] remove redundant dependency --- pkg/gitops/GitOpsConfigService.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/pkg/gitops/GitOpsConfigService.go b/pkg/gitops/GitOpsConfigService.go index fe2e70ec0f..ed3c6bd27b 100644 --- a/pkg/gitops/GitOpsConfigService.go +++ b/pkg/gitops/GitOpsConfigService.go @@ -33,7 +33,6 @@ import ( cluster3 "github.com/argoproj/argo-cd/v2/pkg/apiclient/cluster" bean2 "github.com/devtron-labs/devtron/api/bean" - "github.com/devtron-labs/devtron/client/argocdServer" cluster2 "github.com/devtron-labs/devtron/client/argocdServer/cluster" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/util" @@ -95,8 +94,6 @@ type GitOpsConfigServiceImpl struct { K8sUtil *util4.K8sUtil aCDAuthConfig *util3.ACDAuthConfig clusterService cluster.ClusterService - envService cluster.EnvironmentService - versionService argocdServer.VersionService gitFactory *util.GitFactory chartTemplateService util.ChartTemplateService argoUserService argo.ArgoUserService @@ -106,7 +103,7 @@ type GitOpsConfigServiceImpl struct { func NewGitOpsConfigServiceImpl(Logger *zap.SugaredLogger, globalEnvVariables *util2.GlobalEnvVariables, gitOpsRepository repository.GitOpsConfigRepository, K8sUtil *util4.K8sUtil, aCDAuthConfig *util3.ACDAuthConfig, - clusterService cluster.ClusterService, envService cluster.EnvironmentService, versionService argocdServer.VersionService, + clusterService cluster.ClusterService, gitFactory *util.GitFactory, chartTemplateService util.ChartTemplateService, argoUserService argo.ArgoUserService, clusterServiceCD cluster2.ServiceClient) *GitOpsConfigServiceImpl { return &GitOpsConfigServiceImpl{ randSource: rand.NewSource(time.Now().UnixNano()), @@ -116,8 +113,6 @@ func NewGitOpsConfigServiceImpl(Logger *zap.SugaredLogger, K8sUtil: K8sUtil, aCDAuthConfig: aCDAuthConfig, clusterService: clusterService, - envService: envService, - versionService: versionService, gitFactory: gitFactory, chartTemplateService: chartTemplateService, argoUserService: argoUserService, From ae90f366e81ecb8e3e76e7c5085c1c2dab539efe Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Wed, 17 Jan 2024 14:06:51 +0530 Subject: [PATCH 21/64] moved ChartGroup router and rest handler to respective folder --- Wire.go | 9 +++++---- .../chartGroup}/ChartGroupRestHandler.go | 2 +- .../chartGroup}/ChartGroupRouter.go | 11 +++++------ api/router/router.go | 7 ++++--- wire_gen.go | 5 +++-- 5 files changed, 18 insertions(+), 16 deletions(-) rename api/{restHandler => appStore/chartGroup}/ChartGroupRestHandler.go (99%) rename api/{router => appStore/chartGroup}/ChartGroupRouter.go (83%) diff --git a/Wire.go b/Wire.go index caeb9157b9..459471edd8 100644 --- a/Wire.go +++ b/Wire.go @@ -26,6 +26,7 @@ import ( util4 "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/devtron/api/apiToken" appStoreRestHandler "github.com/devtron-labs/devtron/api/appStore" + chartGroup2 "github.com/devtron-labs/devtron/api/appStore/chartGroup" chartProvider "github.com/devtron-labs/devtron/api/appStore/chartProvider" appStoreDeployment "github.com/devtron-labs/devtron/api/appStore/deployment" appStoreDiscover "github.com/devtron-labs/devtron/api/appStore/discover" @@ -591,10 +592,10 @@ func InitializeApp() (*App, error) { wire.Bind(new(repository4.ChartGroupEntriesRepository), new(*repository4.ChartGroupEntriesRepositoryImpl)), chartGroup.NewChartGroupServiceImpl, wire.Bind(new(chartGroup.ChartGroupService), new(*chartGroup.ChartGroupServiceImpl)), - restHandler.NewChartGroupRestHandlerImpl, - wire.Bind(new(restHandler.ChartGroupRestHandler), new(*restHandler.ChartGroupRestHandlerImpl)), - router.NewChartGroupRouterImpl, - wire.Bind(new(router.ChartGroupRouter), new(*router.ChartGroupRouterImpl)), + chartGroup2.NewChartGroupRestHandlerImpl, + wire.Bind(new(chartGroup2.ChartGroupRestHandler), new(*chartGroup2.ChartGroupRestHandlerImpl)), + chartGroup2.NewChartGroupRouterImpl, + wire.Bind(new(chartGroup2.ChartGroupRouter), new(*chartGroup2.ChartGroupRouterImpl)), repository4.NewChartGroupDeploymentRepositoryImpl, wire.Bind(new(repository4.ChartGroupDeploymentRepository), new(*repository4.ChartGroupDeploymentRepositoryImpl)), diff --git a/api/restHandler/ChartGroupRestHandler.go b/api/appStore/chartGroup/ChartGroupRestHandler.go similarity index 99% rename from api/restHandler/ChartGroupRestHandler.go rename to api/appStore/chartGroup/ChartGroupRestHandler.go index 0e496bea65..5fee0dab7e 100644 --- a/api/restHandler/ChartGroupRestHandler.go +++ b/api/appStore/chartGroup/ChartGroupRestHandler.go @@ -15,7 +15,7 @@ * */ -package restHandler +package chartGroup import ( "encoding/json" diff --git a/api/router/ChartGroupRouter.go b/api/appStore/chartGroup/ChartGroupRouter.go similarity index 83% rename from api/router/ChartGroupRouter.go rename to api/appStore/chartGroup/ChartGroupRouter.go index df28fd471e..48354ce457 100644 --- a/api/router/ChartGroupRouter.go +++ b/api/appStore/chartGroup/ChartGroupRouter.go @@ -15,26 +15,25 @@ * */ -package router +package chartGroup import ( - "github.com/devtron-labs/devtron/api/restHandler" "github.com/gorilla/mux" ) type ChartGroupRouterImpl struct { - ChartGroupRestHandler restHandler.ChartGroupRestHandler + ChartGroupRestHandler ChartGroupRestHandler } type ChartGroupRouter interface { - initChartGroupRouter(helmRouter *mux.Router) + InitChartGroupRouter(helmRouter *mux.Router) } -func NewChartGroupRouterImpl(ChartGroupRestHandler restHandler.ChartGroupRestHandler) *ChartGroupRouterImpl { +func NewChartGroupRouterImpl(ChartGroupRestHandler ChartGroupRestHandler) *ChartGroupRouterImpl { return &ChartGroupRouterImpl{ChartGroupRestHandler: ChartGroupRestHandler} } -func (impl *ChartGroupRouterImpl) initChartGroupRouter(chartGroupRouter *mux.Router) { +func (impl *ChartGroupRouterImpl) InitChartGroupRouter(chartGroupRouter *mux.Router) { chartGroupRouter.Path("/"). HandlerFunc(impl.ChartGroupRestHandler.CreateChartGroup).Methods("POST") chartGroupRouter.Path("/"). diff --git a/api/router/router.go b/api/router/router.go index 558940e3f4..4742d9c060 100644 --- a/api/router/router.go +++ b/api/router/router.go @@ -22,6 +22,7 @@ import ( pubsub2 "github.com/devtron-labs/common-lib/pubsub-lib" "github.com/devtron-labs/devtron/api/apiToken" "github.com/devtron-labs/devtron/api/appStore" + "github.com/devtron-labs/devtron/api/appStore/chartGroup" appStoreDeployment "github.com/devtron-labs/devtron/api/appStore/deployment" "github.com/devtron-labs/devtron/api/auth/sso" "github.com/devtron-labs/devtron/api/auth/user" @@ -81,7 +82,7 @@ type MuxRouter struct { ChartRepositoryRouter chartRepo.ChartRepositoryRouter ReleaseMetricsRouter ReleaseMetricsRouter deploymentGroupRouter DeploymentGroupRouter - chartGroupRouter ChartGroupRouter + chartGroupRouter chartGroup.ChartGroupRouter batchOperationRouter BatchOperationRouter imageScanRouter ImageScanRouter policyRouter PolicyRouter @@ -136,7 +137,7 @@ func NewMuxRouter(logger *zap.SugaredLogger, HelmRouter PipelineTriggerRouter, P ciEventHandler pubsub.CiEventHandler, pubsubClient *pubsub2.PubSubClientServiceImpl, UserRouter user.UserRouter, ChartRefRouter ChartRefRouter, ConfigMapRouter ConfigMapRouter, AppStoreRouter appStore.AppStoreRouter, chartRepositoryRouter chartRepo.ChartRepositoryRouter, ReleaseMetricsRouter ReleaseMetricsRouter, deploymentGroupRouter DeploymentGroupRouter, batchOperationRouter BatchOperationRouter, - chartGroupRouter ChartGroupRouter, imageScanRouter ImageScanRouter, + chartGroupRouter chartGroup.ChartGroupRouter, imageScanRouter ImageScanRouter, policyRouter PolicyRouter, gitOpsConfigRouter GitOpsConfigRouter, dashboardRouter dashboard.DashboardRouter, attributesRouter AttributesRouter, userAttributesRouter UserAttributesRouter, commonRouter CommonRouter, grafanaRouter GrafanaRouter, ssoLoginRouter sso.SsoLoginRouter, telemetryRouter TelemetryRouter, telemetryWatcher telemetry.TelemetryEventClient, bulkUpdateRouter BulkUpdateRouter, webhookListenerRouter WebhookListenerRouter, appRouter AppRouter, coreAppRouter CoreAppRouter, helmAppRouter client.HelmAppRouter, k8sApplicationRouter application.K8sApplicationRouter, @@ -322,7 +323,7 @@ func (r MuxRouter) Init() { r.batchOperationRouter.initBatchOperationRouter(rootRouter) chartGroupRouter := r.Router.PathPrefix("/orchestrator/chart-group").Subrouter() - r.chartGroupRouter.initChartGroupRouter(chartGroupRouter) + r.chartGroupRouter.InitChartGroupRouter(chartGroupRouter) imageScanRouter := r.Router.PathPrefix("/orchestrator/security/scan").Subrouter() r.imageScanRouter.InitImageScanRouter(imageScanRouter) diff --git a/wire_gen.go b/wire_gen.go index a0bd21ec8a..30d30896c5 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -14,6 +14,7 @@ import ( "github.com/devtron-labs/common-lib/utils/k8s" apiToken2 "github.com/devtron-labs/devtron/api/apiToken" "github.com/devtron-labs/devtron/api/appStore" + chartGroup2 "github.com/devtron-labs/devtron/api/appStore/chartGroup" chartProvider2 "github.com/devtron-labs/devtron/api/appStore/chartProvider" "github.com/devtron-labs/devtron/api/appStore/deployment" "github.com/devtron-labs/devtron/api/appStore/discover" @@ -671,8 +672,8 @@ func InitializeApp() (*App, error) { chartGroupEntriesRepositoryImpl := repository11.NewChartGroupEntriesRepositoryImpl(db, sugaredLogger) chartGroupReposotoryImpl := repository11.NewChartGroupReposotoryImpl(db, sugaredLogger) chartGroupServiceImpl := chartGroup.NewChartGroupServiceImpl(chartGroupEntriesRepositoryImpl, chartGroupReposotoryImpl, sugaredLogger, chartGroupDeploymentRepositoryImpl, installedAppRepositoryImpl, appStoreVersionValuesRepositoryImpl, userAuthServiceImpl) - chartGroupRestHandlerImpl := restHandler.NewChartGroupRestHandlerImpl(chartGroupServiceImpl, sugaredLogger, userServiceImpl, enforcerImpl, validate) - chartGroupRouterImpl := router.NewChartGroupRouterImpl(chartGroupRestHandlerImpl) + chartGroupRestHandlerImpl := chartGroup2.NewChartGroupRestHandlerImpl(chartGroupServiceImpl, sugaredLogger, userServiceImpl, enforcerImpl, validate) + chartGroupRouterImpl := chartGroup2.NewChartGroupRouterImpl(chartGroupRestHandlerImpl) scanToolExecutionHistoryMappingRepositoryImpl := security.NewScanToolExecutionHistoryMappingRepositoryImpl(db, sugaredLogger) imageScanServiceImpl := security2.NewImageScanServiceImpl(sugaredLogger, imageScanHistoryRepositoryImpl, imageScanResultRepositoryImpl, imageScanObjectMetaRepositoryImpl, cveStoreRepositoryImpl, imageScanDeployInfoRepositoryImpl, userServiceImpl, teamRepositoryImpl, appRepositoryImpl, environmentServiceImpl, ciArtifactRepositoryImpl, policyServiceImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, scanToolMetadataRepositoryImpl, scanToolExecutionHistoryMappingRepositoryImpl) imageScanRestHandlerImpl := restHandler.NewImageScanRestHandlerImpl(sugaredLogger, imageScanServiceImpl, userServiceImpl, enforcerImpl, enforcerUtilImpl, environmentServiceImpl) From f802e6104ed41067d33553c79b39905436d4dc1b Mon Sep 17 00:00:00 2001 From: nishant Date: Wed, 17 Jan 2024 14:06:53 +0530 Subject: [PATCH 22/64] stage 1 --- .../common/AppStoreDeploymentCommonService.go | 244 ++---------------- .../common/AppStoreDeploymentGitService.go | 204 +++++++++++++++ 2 files changed, 230 insertions(+), 218 deletions(-) create mode 100644 pkg/appStore/deployment/common/AppStoreDeploymentGitService.go diff --git a/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go b/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go index 44ca59836e..48a2760256 100644 --- a/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go +++ b/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go @@ -29,33 +29,39 @@ import ( util2 "github.com/devtron-labs/devtron/pkg/appStore/util" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/go-pg/pg" - "github.com/google/go-github/github" - "github.com/microsoft/azure-devops-go-api/azuredevops" - dirCopy "github.com/otiai10/copy" - "github.com/xanzy/go-gitlab" "go.uber.org/zap" "k8s.io/helm/pkg/chartutil" "k8s.io/helm/pkg/proto/hapi/chart" - "net/http" "os" "path" "path/filepath" - "regexp" "sigs.k8s.io/yaml" ) type AppStoreDeploymentCommonService interface { + //MOve to DB service GetInstalledAppByClusterNamespaceAndName(clusterId int, namespace string, appName string) (*appStoreBean.InstallAppVersionDTO, error) + //move to db service GetInstalledAppByInstalledAppId(installedAppId int) (*appStoreBean.InstallAppVersionDTO, error) + //move to gitops service ParseGitRepoErrorResponse(err error) (bool, error) + //keep here only GetValuesAndRequirementGitConfig(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*util.ChartConfig, *util.ChartConfig, error) + // common for all gitops? CreateChartProxyAndGetPath(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*util.ChartCreateResponse, error) + //move to git service CreateGitOpsRepoAndPushChart(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, builtChartPath string, requirementsConfig *util.ChartConfig, valuesConfig *util.ChartConfig) (*util.ChartGitAttribute, bool, string, error) + //move to gitservice CommitConfigToGit(chartConfig *util.ChartConfig) (gitHash string, err error) + //keep here only GetGitCommitConfig(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, fileString string, filename string) (*util.ChartConfig, error) + //keep here only GetValuesString(chartName, valuesOverrideYaml string) (string, error) + //keep here only GetRequirementsString(appStoreVersionId int) (string, error) + //rethink GenerateManifest(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (manifestResponse *AppStoreManifestResponse, err error) + //gitops service GitOpsOperations(manifestResponse *AppStoreManifestResponse, installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*AppStoreGitOpsResponse, error) GenerateManifestAndPerformGitOperations(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*AppStoreGitOpsResponse, error) } @@ -208,34 +214,6 @@ func (impl AppStoreDeploymentCommonServiceImpl) convert(chart *repository.Instal } } -func (impl AppStoreDeploymentCommonServiceImpl) ParseGitRepoErrorResponse(err error) (bool, error) { - //update values yaml in chart - noTargetFound := false - if err != nil { - if errorResponse, ok := err.(*github.ErrorResponse); ok && errorResponse.Response.StatusCode == http.StatusNotFound { - impl.logger.Errorw("no content found while updating git repo on github, do auto fix", "error", err) - noTargetFound = true - } - if errorResponse, ok := err.(azuredevops.WrappedError); ok && *errorResponse.StatusCode == http.StatusNotFound { - impl.logger.Errorw("no content found while updating git repo on azure, do auto fix", "error", err) - noTargetFound = true - } - if errorResponse, ok := err.(*azuredevops.WrappedError); ok && *errorResponse.StatusCode == http.StatusNotFound { - impl.logger.Errorw("no content found while updating git repo on azure, do auto fix", "error", err) - noTargetFound = true - } - if errorResponse, ok := err.(*gitlab.ErrorResponse); ok && errorResponse.Response.StatusCode == http.StatusNotFound { - impl.logger.Errorw("no content found while updating git repo gitlab, do auto fix", "error", err) - noTargetFound = true - } - if err.Error() == util.BITBUCKET_REPO_NOT_FOUND_ERROR { - impl.logger.Errorw("no content found while updating git repo bitbucket, do auto fix", "error", err) - noTargetFound = true - } - } - return noTargetFound, err -} - func (impl AppStoreDeploymentCommonServiceImpl) GetValuesString(chartName, valuesOverrideYaml string) (string, error) { ValuesOverrideByte, err := yaml.YAMLToJSON([]byte(valuesOverrideYaml)) @@ -347,27 +325,6 @@ func (impl AppStoreDeploymentCommonServiceImpl) GetValuesAndRequirementGitConfig return valuesConfig, RequirementConfig, nil } -// CreateChartProxyAndGetPath parse chart in local directory and returns path of local dir and values.yaml -func (impl AppStoreDeploymentCommonServiceImpl) CreateChartProxyAndGetPath(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*util.ChartCreateResponse, error) { - - ChartCreateResponse := &util.ChartCreateResponse{} - template := appStoreBean.CHART_PROXY_TEMPLATE - chartPath := path.Join(string(impl.refChartDir), template) - valid, err := chartutil.IsChartDir(chartPath) - if err != nil || !valid { - impl.logger.Errorw("invalid base chart", "dir", chartPath, "err", err) - return ChartCreateResponse, err - } - chartCreateRequest := ParseChartCreateRequest(installAppVersionRequest, chartPath) - chartCreateResponse, err := impl.chartTemplateService.BuildChartProxyForHelmApps(chartCreateRequest) - if err != nil { - impl.logger.Errorw("Error in building chart proxy", "err", err) - return chartCreateResponse, err - } - return chartCreateResponse, nil - -} - func (impl AppStoreDeploymentCommonServiceImpl) GenerateManifest(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (manifestResponse *AppStoreManifestResponse, err error) { manifestResponse = &AppStoreManifestResponse{} @@ -390,17 +347,6 @@ func (impl AppStoreDeploymentCommonServiceImpl) GenerateManifest(installAppVersi return manifestResponse, nil } -//func (impl AppStoreDeploymentCommonServiceImpl) GenerateManifestAndPerformGitOps(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*util.ChartGitAttribute, error) { -// -// manifestResponse, err := impl.GenerateManifest(installAppVersionRequest) -// if err != nil { -// impl.logger.Errorw("Error in generating manifest for gitops step", "err", err) -// return nil, err -// } -// impl. -// -//} - // CreateGitOpsRepo creates a gitOps repo with readme func (impl AppStoreDeploymentCommonServiceImpl) CreateGitOpsRepo(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (string, bool, error) { @@ -438,72 +384,6 @@ func (impl AppStoreDeploymentCommonServiceImpl) CreateGitOpsRepo(installAppVersi return repoUrl, isNew, err } -// PushChartToGitopsRepo pushes built chart to gitOps repo -func (impl AppStoreDeploymentCommonServiceImpl) PushChartToGitopsRepo(PushChartToGitRequest *appStoreBean.PushChartToGitRequestDTO, requirementsConfig *util.ChartConfig, valuesConfig *util.ChartConfig) (*util.ChartGitAttribute, string, error) { - space := regexp.MustCompile(`\s+`) - appStoreName := space.ReplaceAllString(PushChartToGitRequest.ChartAppStoreName, "-") - chartDir := fmt.Sprintf("%s-%s", PushChartToGitRequest.AppName, impl.chartTemplateService.GetDir()) - clonedDir := impl.gitFactory.GitService.GetCloneDirectory(chartDir) - if _, err := os.Stat(clonedDir); os.IsNotExist(err) { - clonedDir, err = impl.gitFactory.GitService.Clone(PushChartToGitRequest.RepoURL, chartDir) - if err != nil { - impl.logger.Errorw("error in cloning repo", "url", PushChartToGitRequest.RepoURL, "err", err) - return nil, "", err - } - } else { - err = impl.chartTemplateService.GitPull(clonedDir, PushChartToGitRequest.RepoURL, appStoreName) - if err != nil { - return nil, "", err - } - } - acdAppName := fmt.Sprintf("%s-%s", PushChartToGitRequest.AppName, PushChartToGitRequest.EnvName) - dir := filepath.Join(clonedDir, acdAppName) - err := os.MkdirAll(dir, os.ModePerm) - if err != nil { - impl.logger.Errorw("error in making dir", "err", err) - return nil, "", err - } - err = dirCopy.Copy(PushChartToGitRequest.TempChartRefDir, dir) - if err != nil { - impl.logger.Errorw("error copying dir", "err", err) - return nil, "", err - } - err = impl.AddConfigFileToChart(requirementsConfig, dir, clonedDir) - if err != nil { - impl.logger.Errorw("error in adding requirements.yaml to chart", "err", err, "appName", PushChartToGitRequest.AppName) - return nil, "", err - } - err = impl.AddConfigFileToChart(valuesConfig, dir, clonedDir) - if err != nil { - impl.logger.Errorw("error in adding values.yaml to chart", "err", err, "appName", PushChartToGitRequest.AppName) - return nil, "", err - } - userEmailId, userName := impl.chartTemplateService.GetUserEmailIdAndNameForGitOpsCommit(PushChartToGitRequest.UserId) - commit, err := impl.gitFactory.GitService.CommitAndPushAllChanges(clonedDir, "first commit", userName, userEmailId) - if err != nil { - impl.logger.Errorw("error in pushing git", "err", err) - impl.logger.Warn("re-trying, taking pull and then push again") - err = impl.chartTemplateService.GitPull(clonedDir, PushChartToGitRequest.RepoURL, acdAppName) - if err != nil { - impl.logger.Errorw("error in git pull", "err", err, "appName", acdAppName) - return nil, "", err - } - err = dirCopy.Copy(PushChartToGitRequest.TempChartRefDir, dir) - if err != nil { - impl.logger.Errorw("error copying dir", "err", err) - return nil, "", err - } - commit, err = impl.gitFactory.GitService.CommitAndPushAllChanges(clonedDir, "first commit", userName, userEmailId) - if err != nil { - impl.logger.Errorw("error in pushing git", "err", err) - return nil, "", err - } - } - impl.logger.Debugw("template committed", "url", PushChartToGitRequest.RepoURL, "commit", commit) - defer impl.chartTemplateService.CleanDir(clonedDir) - return &util.ChartGitAttribute{RepoUrl: PushChartToGitRequest.RepoURL, ChartLocation: acdAppName}, commit, err -} - // AddConfigFileToChart will override requirements.yaml file in chart func (impl AppStoreDeploymentCommonServiceImpl) AddConfigFileToChart(config *util.ChartConfig, dir string, clonedDir string) error { filePath := filepath.Join(clonedDir, config.FileName) @@ -527,95 +407,23 @@ func (impl AppStoreDeploymentCommonServiceImpl) AddConfigFileToChart(config *uti return nil } -// CreateGitOpsRepoAndPushChart is a wrapper for creating gitops repo and pushing chart to created repo -func (impl AppStoreDeploymentCommonServiceImpl) CreateGitOpsRepoAndPushChart(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, builtChartPath string, requirementsConfig *util.ChartConfig, valuesConfig *util.ChartConfig) (*util.ChartGitAttribute, bool, string, error) { - repoURL, isNew, err := impl.CreateGitOpsRepo(installAppVersionRequest) - if err != nil { - impl.logger.Errorw("Error in creating gitops repo for ", "appName", installAppVersionRequest.AppName, "err", err) - return nil, false, "", err - } - pushChartToGitRequest := ParseChartGitPushRequest(installAppVersionRequest, repoURL, builtChartPath) - chartGitAttribute, commitHash, err := impl.PushChartToGitopsRepo(pushChartToGitRequest, requirementsConfig, valuesConfig) - if err != nil { - impl.logger.Errorw("error in pushing chart to git", "err", err) - return nil, false, "", err - } - return chartGitAttribute, isNew, commitHash, err -} +// CreateChartProxyAndGetPath parse chart in local directory and returns path of local dir and values.yaml +func (impl AppStoreDeploymentCommonServiceImpl) CreateChartProxyAndGetPath(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*util.ChartCreateResponse, error) { -// CommitConfigToGit is used for committing values.yaml and requirements.yaml file config -func (impl AppStoreDeploymentCommonServiceImpl) CommitConfigToGit(chartConfig *util.ChartConfig) (string, error) { - gitOpsConfigBitbucket, err := impl.gitOpsConfigRepository.GetGitOpsConfigByProvider(util.BITBUCKET_PROVIDER) - if err != nil { - if err == pg.ErrNoRows { - gitOpsConfigBitbucket.BitBucketWorkspaceId = "" - } else { - return "", err - } - } - gitOpsConfig := &bean.GitOpsConfigDto{BitBucketWorkspaceId: gitOpsConfigBitbucket.BitBucketWorkspaceId} - gitHash, _, err := impl.gitFactory.Client.CommitValues(chartConfig, gitOpsConfig) - if err != nil { - impl.logger.Errorw("error in git commit", "err", err) - return "", err + ChartCreateResponse := &util.ChartCreateResponse{} + template := appStoreBean.CHART_PROXY_TEMPLATE + chartPath := path.Join(string(impl.refChartDir), template) + valid, err := chartutil.IsChartDir(chartPath) + if err != nil || !valid { + impl.logger.Errorw("invalid base chart", "dir", chartPath, "err", err) + return ChartCreateResponse, err } - return gitHash, nil -} - -func (impl AppStoreDeploymentCommonServiceImpl) GitOpsOperations(manifestResponse *AppStoreManifestResponse, installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*AppStoreGitOpsResponse, error) { - appStoreGitOpsResponse := &AppStoreGitOpsResponse{} - chartGitAttribute, isNew, githash, err := impl.CreateGitOpsRepoAndPushChart(installAppVersionRequest, manifestResponse.ChartResponse.BuiltChartPath, manifestResponse.RequirementsConfig, manifestResponse.ValuesConfig) + chartCreateRequest := ParseChartCreateRequest(installAppVersionRequest, chartPath) + chartCreateResponse, err := impl.chartTemplateService.BuildChartProxyForHelmApps(chartCreateRequest) if err != nil { - impl.logger.Errorw("Error in pushing chart to git", "err", err) - return appStoreGitOpsResponse, err - } - space := regexp.MustCompile(`\s+`) - appStoreName := space.ReplaceAllString(installAppVersionRequest.AppName, "-") - clonedDir := impl.gitFactory.GitWorkingDir + "" + appStoreName - - // Checking this is the first time chart has been pushed , if yes requirements.yaml has been already pushed with chart as there was sync-delay with github api. - // step-2 commit dependencies and values in git - if !isNew { - _, err = impl.CommitConfigToGit(manifestResponse.RequirementsConfig) - if err != nil { - impl.logger.Errorw("error in committing dependency config to git", "err", err) - return appStoreGitOpsResponse, err - } - err = impl.chartTemplateService.GitPull(clonedDir, chartGitAttribute.RepoUrl, appStoreName) - if err != nil { - impl.logger.Errorw("error in git pull", "err", err) - return appStoreGitOpsResponse, err - } - - githash, err = impl.CommitConfigToGit(manifestResponse.ValuesConfig) - if err != nil { - impl.logger.Errorw("error in committing values config to git", "err", err) - return appStoreGitOpsResponse, err - } - err = impl.chartTemplateService.GitPull(clonedDir, chartGitAttribute.RepoUrl, appStoreName) - if err != nil { - impl.logger.Errorw("error in git pull", "err", err) - return appStoreGitOpsResponse, err - } + impl.logger.Errorw("Error in building chart proxy", "err", err) + return chartCreateResponse, err } - appStoreGitOpsResponse.ChartGitAttribute = chartGitAttribute - appStoreGitOpsResponse.GitHash = githash - return appStoreGitOpsResponse, nil -} + return chartCreateResponse, nil -func (impl AppStoreDeploymentCommonServiceImpl) GenerateManifestAndPerformGitOperations(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*AppStoreGitOpsResponse, error) { - appStoreGitOpsResponse := &AppStoreGitOpsResponse{} - manifest, err := impl.GenerateManifest(installAppVersionRequest) - if err != nil { - impl.logger.Errorw("error in performing manifest and git operations", "err", err) - return nil, err - } - gitOpsResponse, err := impl.GitOpsOperations(manifest, installAppVersionRequest) - if err != nil { - impl.logger.Errorw("error in performing gitops operation", "err", err) - return nil, err - } - installAppVersionRequest.GitHash = gitOpsResponse.GitHash - appStoreGitOpsResponse.ChartGitAttribute = gitOpsResponse.ChartGitAttribute - return appStoreGitOpsResponse, nil } diff --git a/pkg/appStore/deployment/common/AppStoreDeploymentGitService.go b/pkg/appStore/deployment/common/AppStoreDeploymentGitService.go new file mode 100644 index 0000000000..3ffb86cf9f --- /dev/null +++ b/pkg/appStore/deployment/common/AppStoreDeploymentGitService.go @@ -0,0 +1,204 @@ +package appStoreDeploymentCommon + +import ( + "fmt" + "github.com/devtron-labs/devtron/api/bean" + "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/appStore/bean" + "github.com/go-pg/pg" + "github.com/google/go-github/github" + "github.com/microsoft/azure-devops-go-api/azuredevops" + copy2 "github.com/otiai10/copy" + "github.com/xanzy/go-gitlab" + "net/http" + "os" + "path/filepath" + "regexp" +) + +func (impl AppStoreDeploymentCommonServiceImpl) ParseGitRepoErrorResponse(err error) (bool, error) { + //update values yaml in chart + noTargetFound := false + if err != nil { + if errorResponse, ok := err.(*github.ErrorResponse); ok && errorResponse.Response.StatusCode == http.StatusNotFound { + impl.logger.Errorw("no content found while updating git repo on github, do auto fix", "error", err) + noTargetFound = true + } + if errorResponse, ok := err.(azuredevops.WrappedError); ok && *errorResponse.StatusCode == http.StatusNotFound { + impl.logger.Errorw("no content found while updating git repo on azure, do auto fix", "error", err) + noTargetFound = true + } + if errorResponse, ok := err.(*azuredevops.WrappedError); ok && *errorResponse.StatusCode == http.StatusNotFound { + impl.logger.Errorw("no content found while updating git repo on azure, do auto fix", "error", err) + noTargetFound = true + } + if errorResponse, ok := err.(*gitlab.ErrorResponse); ok && errorResponse.Response.StatusCode == http.StatusNotFound { + impl.logger.Errorw("no content found while updating git repo gitlab, do auto fix", "error", err) + noTargetFound = true + } + if err.Error() == util.BITBUCKET_REPO_NOT_FOUND_ERROR { + impl.logger.Errorw("no content found while updating git repo bitbucket, do auto fix", "error", err) + noTargetFound = true + } + } + return noTargetFound, err +} + +// CreateGitOpsRepoAndPushChart is a wrapper for creating gitops repo and pushing chart to created repo +func (impl AppStoreDeploymentCommonServiceImpl) CreateGitOpsRepoAndPushChart(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, builtChartPath string, requirementsConfig *util.ChartConfig, valuesConfig *util.ChartConfig) (*util.ChartGitAttribute, bool, string, error) { + repoURL, isNew, err := impl.CreateGitOpsRepo(installAppVersionRequest) + if err != nil { + impl.logger.Errorw("Error in creating gitops repo for ", "appName", installAppVersionRequest.AppName, "err", err) + return nil, false, "", err + } + pushChartToGitRequest := ParseChartGitPushRequest(installAppVersionRequest, repoURL, builtChartPath) + chartGitAttribute, commitHash, err := impl.PushChartToGitopsRepo(pushChartToGitRequest, requirementsConfig, valuesConfig) + if err != nil { + impl.logger.Errorw("error in pushing chart to git", "err", err) + return nil, false, "", err + } + return chartGitAttribute, isNew, commitHash, err +} + +// CommitConfigToGit is used for committing values.yaml and requirements.yaml file config +func (impl AppStoreDeploymentCommonServiceImpl) CommitConfigToGit(chartConfig *util.ChartConfig) (string, error) { + gitOpsConfigBitbucket, err := impl.gitOpsConfigRepository.GetGitOpsConfigByProvider(util.BITBUCKET_PROVIDER) + if err != nil { + if err == pg.ErrNoRows { + gitOpsConfigBitbucket.BitBucketWorkspaceId = "" + } else { + return "", err + } + } + gitOpsConfig := &bean.GitOpsConfigDto{BitBucketWorkspaceId: gitOpsConfigBitbucket.BitBucketWorkspaceId} + gitHash, _, err := impl.gitFactory.Client.CommitValues(chartConfig, gitOpsConfig) + if err != nil { + impl.logger.Errorw("error in git commit", "err", err) + return "", err + } + return gitHash, nil +} + +func (impl AppStoreDeploymentCommonServiceImpl) GitOpsOperations(manifestResponse *AppStoreManifestResponse, installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*AppStoreGitOpsResponse, error) { + appStoreGitOpsResponse := &AppStoreGitOpsResponse{} + chartGitAttribute, isNew, githash, err := impl.CreateGitOpsRepoAndPushChart(installAppVersionRequest, manifestResponse.ChartResponse.BuiltChartPath, manifestResponse.RequirementsConfig, manifestResponse.ValuesConfig) + if err != nil { + impl.logger.Errorw("Error in pushing chart to git", "err", err) + return appStoreGitOpsResponse, err + } + space := regexp.MustCompile(`\s+`) + appStoreName := space.ReplaceAllString(installAppVersionRequest.AppName, "-") + clonedDir := impl.gitFactory.GitWorkingDir + "" + appStoreName + + // Checking this is the first time chart has been pushed , if yes requirements.yaml has been already pushed with chart as there was sync-delay with github api. + // step-2 commit dependencies and values in git + if !isNew { + _, err = impl.CommitConfigToGit(manifestResponse.RequirementsConfig) + if err != nil { + impl.logger.Errorw("error in committing dependency config to git", "err", err) + return appStoreGitOpsResponse, err + } + err = impl.chartTemplateService.GitPull(clonedDir, chartGitAttribute.RepoUrl, appStoreName) + if err != nil { + impl.logger.Errorw("error in git pull", "err", err) + return appStoreGitOpsResponse, err + } + + githash, err = impl.CommitConfigToGit(manifestResponse.ValuesConfig) + if err != nil { + impl.logger.Errorw("error in committing values config to git", "err", err) + return appStoreGitOpsResponse, err + } + err = impl.chartTemplateService.GitPull(clonedDir, chartGitAttribute.RepoUrl, appStoreName) + if err != nil { + impl.logger.Errorw("error in git pull", "err", err) + return appStoreGitOpsResponse, err + } + } + appStoreGitOpsResponse.ChartGitAttribute = chartGitAttribute + appStoreGitOpsResponse.GitHash = githash + return appStoreGitOpsResponse, nil +} + +func (impl AppStoreDeploymentCommonServiceImpl) GenerateManifestAndPerformGitOperations(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*AppStoreGitOpsResponse, error) { + appStoreGitOpsResponse := &AppStoreGitOpsResponse{} + manifest, err := impl.GenerateManifest(installAppVersionRequest) + if err != nil { + impl.logger.Errorw("error in performing manifest and git operations", "err", err) + return nil, err + } + gitOpsResponse, err := impl.GitOpsOperations(manifest, installAppVersionRequest) + if err != nil { + impl.logger.Errorw("error in performing gitops operation", "err", err) + return nil, err + } + installAppVersionRequest.GitHash = gitOpsResponse.GitHash + appStoreGitOpsResponse.ChartGitAttribute = gitOpsResponse.ChartGitAttribute + return appStoreGitOpsResponse, nil +} + +// PushChartToGitopsRepo pushes built chart to gitOps repo +func (impl AppStoreDeploymentCommonServiceImpl) PushChartToGitopsRepo(PushChartToGitRequest *appStoreBean.PushChartToGitRequestDTO, requirementsConfig *util.ChartConfig, valuesConfig *util.ChartConfig) (*util.ChartGitAttribute, string, error) { + space := regexp.MustCompile(`\s+`) + appStoreName := space.ReplaceAllString(PushChartToGitRequest.ChartAppStoreName, "-") + chartDir := fmt.Sprintf("%s-%s", PushChartToGitRequest.AppName, impl.chartTemplateService.GetDir()) + clonedDir := impl.gitFactory.GitService.GetCloneDirectory(chartDir) + if _, err := os.Stat(clonedDir); os.IsNotExist(err) { + clonedDir, err = impl.gitFactory.GitService.Clone(PushChartToGitRequest.RepoURL, chartDir) + if err != nil { + impl.logger.Errorw("error in cloning repo", "url", PushChartToGitRequest.RepoURL, "err", err) + return nil, "", err + } + } else { + err = impl.chartTemplateService.GitPull(clonedDir, PushChartToGitRequest.RepoURL, appStoreName) + if err != nil { + return nil, "", err + } + } + acdAppName := fmt.Sprintf("%s-%s", PushChartToGitRequest.AppName, PushChartToGitRequest.EnvName) + dir := filepath.Join(clonedDir, acdAppName) + err := os.MkdirAll(dir, os.ModePerm) + if err != nil { + impl.logger.Errorw("error in making dir", "err", err) + return nil, "", err + } + err = copy2.Copy(PushChartToGitRequest.TempChartRefDir, dir) + if err != nil { + impl.logger.Errorw("error copying dir", "err", err) + return nil, "", err + } + err = impl.AddConfigFileToChart(requirementsConfig, dir, clonedDir) + if err != nil { + impl.logger.Errorw("error in adding requirements.yaml to chart", "err", err, "appName", PushChartToGitRequest.AppName) + return nil, "", err + } + err = impl.AddConfigFileToChart(valuesConfig, dir, clonedDir) + if err != nil { + impl.logger.Errorw("error in adding values.yaml to chart", "err", err, "appName", PushChartToGitRequest.AppName) + return nil, "", err + } + userEmailId, userName := impl.chartTemplateService.GetUserEmailIdAndNameForGitOpsCommit(PushChartToGitRequest.UserId) + commit, err := impl.gitFactory.GitService.CommitAndPushAllChanges(clonedDir, "first commit", userName, userEmailId) + if err != nil { + impl.logger.Errorw("error in pushing git", "err", err) + impl.logger.Warn("re-trying, taking pull and then push again") + err = impl.chartTemplateService.GitPull(clonedDir, PushChartToGitRequest.RepoURL, acdAppName) + if err != nil { + impl.logger.Errorw("error in git pull", "err", err, "appName", acdAppName) + return nil, "", err + } + err = copy2.Copy(PushChartToGitRequest.TempChartRefDir, dir) + if err != nil { + impl.logger.Errorw("error copying dir", "err", err) + return nil, "", err + } + commit, err = impl.gitFactory.GitService.CommitAndPushAllChanges(clonedDir, "first commit", userName, userEmailId) + if err != nil { + impl.logger.Errorw("error in pushing git", "err", err) + return nil, "", err + } + } + impl.logger.Debugw("template committed", "url", PushChartToGitRequest.RepoURL, "commit", commit) + defer impl.chartTemplateService.CleanDir(clonedDir) + return &util.ChartGitAttribute{RepoUrl: PushChartToGitRequest.RepoURL, ChartLocation: acdAppName}, commit, err +} From 9c0f2b882a32a39b88c3c96b046dc5e849aacf04 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Thu, 18 Jan 2024 12:28:28 +0530 Subject: [PATCH 23/64] gitOps refactoring --- Wire.go | 4 +- api/restHandler/CommonRestHanlder.go | 25 +- api/restHandler/GitOpsConfigRestHandler.go | 5 +- cmd/external-app/wire.go | 3 +- cmd/external-app/wire_gen.go | 15 +- internal/util/ChartService.go | 404 +----------------- internal/util/GitService.go | 1 + pkg/app/AppService.go | 19 +- pkg/app/ManifestPushService.go | 37 +- pkg/appClone/AppCloneService.go | 16 +- .../common/AppStoreDeploymentCommonService.go | 67 ++- .../AppStoreDeploymentFullModeService.go | 274 ++---------- .../service/AppStoreDeploymentService.go | 21 +- .../deployment/service/InstalledAppService.go | 27 +- .../tool/AppStoreDeploymentHelmService.go | 11 +- .../gitops/AppStoreDeploymentArgoCdService.go | 29 +- pkg/chart/ChartService.go | 26 +- pkg/commonService/CommonService.go | 3 - pkg/deployment/gitOps/common/bean/bean.go | 6 + .../gitOps/config/GitOpsConfigReadService.go | 133 ++++++ pkg/deployment/gitOps/config/bean/bean.go | 14 + .../remote/GitOpsRemoteOperationService.go | 316 ++++++++++++++ pkg/deployment/gitOps/remote/bean/bean.go | 7 + pkg/deployment/gitOps/wire_gitOps.go | 15 + pkg/deployment/wire_deployment.go | 14 + pkg/gitops/GitOpsConfigService.go | 104 ++--- .../AppDeploymentTypeChangeManager.go | 31 +- .../DeploymentPipelineConfigService.go | 54 +-- pkg/pipeline/WorkflowDagExecutor.go | 24 +- wire_gen.go | 60 +-- 30 files changed, 842 insertions(+), 923 deletions(-) create mode 100644 pkg/deployment/gitOps/common/bean/bean.go create mode 100644 pkg/deployment/gitOps/config/GitOpsConfigReadService.go create mode 100644 pkg/deployment/gitOps/config/bean/bean.go create mode 100644 pkg/deployment/gitOps/remote/GitOpsRemoteOperationService.go create mode 100644 pkg/deployment/gitOps/remote/bean/bean.go create mode 100644 pkg/deployment/gitOps/wire_gitOps.go create mode 100644 pkg/deployment/wire_deployment.go diff --git a/Wire.go b/Wire.go index 0dc8fefc60..7848ba4221 100644 --- a/Wire.go +++ b/Wire.go @@ -96,7 +96,7 @@ import ( chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" "github.com/devtron-labs/devtron/pkg/commonService" delete2 "github.com/devtron-labs/devtron/pkg/delete" - "github.com/devtron-labs/devtron/pkg/deployment/manifest" + deployment2 "github.com/devtron-labs/devtron/pkg/deployment" "github.com/devtron-labs/devtron/pkg/deploymentGroup" "github.com/devtron-labs/devtron/pkg/devtronResource" repository9 "github.com/devtron-labs/devtron/pkg/devtronResource/repository" @@ -157,7 +157,7 @@ func InitializeApp() (*App, error) { apiToken.ApiTokenWireSet, webhookHelm.WebhookHelmWireSet, terminal.TerminalWireSet, - manifest.DeploymentManifestWireSet, + deployment2.DeploymentWireSet, // -------wireset end ---------- //------- diff --git a/api/restHandler/CommonRestHanlder.go b/api/restHandler/CommonRestHanlder.go index 41007976c8..3f95c46cf3 100644 --- a/api/restHandler/CommonRestHanlder.go +++ b/api/restHandler/CommonRestHanlder.go @@ -21,12 +21,9 @@ import ( "net/http" "github.com/devtron-labs/devtron/api/restHandler/common" - "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" "github.com/devtron-labs/devtron/pkg/auth/user" "github.com/devtron-labs/devtron/pkg/commonService" - "github.com/devtron-labs/devtron/pkg/gitops" "go.uber.org/zap" - "gopkg.in/go-playground/validator.v9" ) type CommonRestHanlder interface { @@ -34,25 +31,19 @@ type CommonRestHanlder interface { } type CommonRestHanlderImpl struct { - logger *zap.SugaredLogger - gitOpsConfigService gitops.GitOpsConfigService - userAuthService user.UserService - validator *validator.Validate - enforcer casbin.Enforcer - commonService commonService.CommonService + logger *zap.SugaredLogger + userAuthService user.UserService + commonService commonService.CommonService } func NewCommonRestHanlderImpl( logger *zap.SugaredLogger, - gitOpsConfigService gitops.GitOpsConfigService, userAuthService user.UserService, - validator *validator.Validate, enforcer casbin.Enforcer, commonService commonService.CommonService) *CommonRestHanlderImpl { + userAuthService user.UserService, + commonService commonService.CommonService) *CommonRestHanlderImpl { return &CommonRestHanlderImpl{ - logger: logger, - gitOpsConfigService: gitOpsConfigService, - userAuthService: userAuthService, - validator: validator, - enforcer: enforcer, - commonService: commonService, + logger: logger, + userAuthService: userAuthService, + commonService: commonService, } } diff --git a/api/restHandler/GitOpsConfigRestHandler.go b/api/restHandler/GitOpsConfigRestHandler.go index 120d310a8b..da9130f6c4 100644 --- a/api/restHandler/GitOpsConfigRestHandler.go +++ b/api/restHandler/GitOpsConfigRestHandler.go @@ -25,7 +25,6 @@ import ( bean2 "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/api/restHandler/common" - "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" "github.com/devtron-labs/devtron/pkg/auth/user" "github.com/devtron-labs/devtron/pkg/gitops" @@ -52,13 +51,12 @@ type GitOpsConfigRestHandlerImpl struct { validator *validator.Validate enforcer casbin.Enforcer teamService team.TeamService - gitOpsRepository repository.GitOpsConfigRepository } func NewGitOpsConfigRestHandlerImpl( logger *zap.SugaredLogger, gitOpsConfigService gitops.GitOpsConfigService, userAuthService user.UserService, - validator *validator.Validate, enforcer casbin.Enforcer, teamService team.TeamService, gitOpsRepository repository.GitOpsConfigRepository) *GitOpsConfigRestHandlerImpl { + validator *validator.Validate, enforcer casbin.Enforcer, teamService team.TeamService) *GitOpsConfigRestHandlerImpl { return &GitOpsConfigRestHandlerImpl{ logger: logger, gitOpsConfigService: gitOpsConfigService, @@ -66,7 +64,6 @@ func NewGitOpsConfigRestHandlerImpl( validator: validator, enforcer: enforcer, teamService: teamService, - gitOpsRepository: gitOpsRepository, } } diff --git a/cmd/external-app/wire.go b/cmd/external-app/wire.go index 45c6a0f225..13b3901b88 100644 --- a/cmd/external-app/wire.go +++ b/cmd/external-app/wire.go @@ -43,6 +43,7 @@ import ( appStoreDeploymentGitopsTool "github.com/devtron-labs/devtron/pkg/appStore/deployment/tool/gitops" "github.com/devtron-labs/devtron/pkg/attributes" delete2 "github.com/devtron-labs/devtron/pkg/delete" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps" "github.com/devtron-labs/devtron/pkg/kubernetesResourceAuditLogs" repository2 "github.com/devtron-labs/devtron/pkg/kubernetesResourceAuditLogs/repository" "github.com/devtron-labs/devtron/pkg/pipeline" @@ -79,7 +80,7 @@ func InitializeApp() (*App, error) { apiToken.ApiTokenWireSet, webhookHelm.WebhookHelmWireSet, terminal.TerminalWireSet, - + gitOps.GitOpsWireSet, NewApp, NewMuxRouter, util3.GetGlobalEnvVariables, diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index de881eb3d0..f664986f3c 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -66,6 +66,8 @@ import ( repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/clusterTerminalAccess" delete2 "github.com/devtron-labs/devtron/pkg/delete" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote" "github.com/devtron-labs/devtron/pkg/externalLink" "github.com/devtron-labs/devtron/pkg/genericNotes" repository6 "github.com/devtron-labs/devtron/pkg/genericNotes/repository" @@ -95,7 +97,7 @@ import ( // Injectors from wire.go: func InitializeApp() (*App, error) { - config, err := sql.GetConfig() + sqlConfig, err := sql.GetConfig() if err != nil { return nil, err } @@ -103,7 +105,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - db, err := sql.NewDbConnection(config, sugaredLogger) + db, err := sql.NewDbConnection(sqlConfig, sugaredLogger) if err != nil { return nil, err } @@ -228,6 +230,7 @@ func InitializeApp() (*App, error) { return nil, err } dashboardRouterImpl := dashboard.NewDashboardRouterImpl(sugaredLogger, dashboardConfig) + chartTemplateServiceImpl := util.NewChartTemplateServiceImpl(sugaredLogger) gitOpsConfigRepositoryImpl := repository3.NewGitOpsConfigRepositoryImpl(sugaredLogger, db) gitCliUtil := util.NewGitCliUtil(sugaredLogger) gitFactory, err := util.NewGitFactory(sugaredLogger, gitOpsConfigRepositoryImpl, gitCliUtil) @@ -238,9 +241,9 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - chartRepositoryImpl := chartRepoRepository.NewChartRepository(db) - chartTemplateServiceImpl := util.NewChartTemplateServiceImpl(sugaredLogger, gitFactory, globalEnvVariables, gitOpsConfigRepositoryImpl, userRepositoryImpl, chartRepositoryImpl) - appStoreDeploymentCommonServiceImpl := appStoreDeploymentCommon.NewAppStoreDeploymentCommonServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, chartTemplateServiceImpl, gitFactory, gitOpsConfigRepositoryImpl) + gitOpsConfigReadServiceImpl := config.NewGitOpsConfigReadServiceImpl(sugaredLogger, gitOpsConfigRepositoryImpl, userServiceImpl, globalEnvVariables) + gitOpsRemoteOperationServiceImpl := remote.NewGitOpsRemoteOperationServiceImpl(sugaredLogger, gitFactory, gitOpsConfigReadServiceImpl, chartTemplateServiceImpl) + appStoreDeploymentCommonServiceImpl := appStoreDeploymentCommon.NewAppStoreDeploymentCommonServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, chartTemplateServiceImpl, gitFactory, gitOpsConfigReadServiceImpl, gitOpsRemoteOperationServiceImpl) attributesServiceImpl := attributes.NewAttributesServiceImpl(sugaredLogger, attributesRepositoryImpl) helmAppRestHandlerImpl := client2.NewHelmAppRestHandlerImpl(sugaredLogger, helmAppServiceImpl, enforcerImpl, clusterServiceImpl, enforcerUtilHelmImpl, appStoreDeploymentCommonServiceImpl, userServiceImpl, attributesServiceImpl, serverEnvConfigServerEnvConfig) helmAppRouterImpl := client2.NewHelmAppRouterImpl(helmAppRestHandlerImpl) @@ -281,7 +284,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - appStoreDeploymentServiceImpl := service3.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, clusterInstalledAppsRepositoryImpl, appRepositoryImpl, appStoreDeploymentHelmServiceImpl, appStoreDeploymentHelmServiceImpl, environmentServiceImpl, clusterServiceImpl, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, globalEnvVariables, installedAppVersionHistoryRepositoryImpl, gitOpsConfigRepositoryImpl, attributesServiceImpl, deploymentServiceTypeConfig, chartTemplateServiceImpl, acdConfig) + appStoreDeploymentServiceImpl := service3.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, clusterInstalledAppsRepositoryImpl, appRepositoryImpl, appStoreDeploymentHelmServiceImpl, appStoreDeploymentHelmServiceImpl, environmentServiceImpl, clusterServiceImpl, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, globalEnvVariables, installedAppVersionHistoryRepositoryImpl, gitOpsConfigRepositoryImpl, deploymentServiceTypeConfig, acdConfig, gitOpsConfigReadServiceImpl) appStoreDeploymentRestHandlerImpl := appStoreDeployment.NewAppStoreDeploymentRestHandlerImpl(sugaredLogger, userServiceImpl, enforcerImpl, enforcerUtilImpl, enforcerUtilHelmImpl, appStoreDeploymentServiceImpl, validate, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, helmUserServiceImpl, attributesServiceImpl) appStoreDeploymentRouterImpl := appStoreDeployment.NewAppStoreDeploymentRouterImpl(appStoreDeploymentRestHandlerImpl) chartProviderServiceImpl := chartProvider.NewChartProviderServiceImpl(sugaredLogger, chartRepoRepositoryImpl, chartRepositoryServiceImpl, dockerArtifactStoreRepositoryImpl, ociRegistryConfigRepositoryImpl) diff --git a/internal/util/ChartService.go b/internal/util/ChartService.go index 2deb276c30..f904ed4490 100644 --- a/internal/util/ChartService.go +++ b/internal/util/ChartService.go @@ -25,20 +25,11 @@ import ( "math/rand" "os" "path/filepath" - "regexp" "strconv" "strings" "time" dockerRegistryRepository "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" - repository2 "github.com/devtron-labs/devtron/pkg/auth/user/repository" - - "github.com/devtron-labs/devtron/api/bean" - "github.com/devtron-labs/devtron/internal/sql/repository" - appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" - chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" - "github.com/devtron-labs/devtron/util" - "github.com/go-pg/pg" dirCopy "github.com/otiai10/copy" "go.opentelemetry.io/otel" "go.uber.org/zap" @@ -65,35 +56,21 @@ type ChartCreateResponse struct { } type ChartTemplateService interface { - FetchValuesFromReferenceChart(chartMetaData *chart.Metadata, refChartLocation string, templateName string, userId int32, pipelineStrategyPath string) (*ChartValues, *ChartGitAttribute, error) + FetchValuesFromReferenceChart(chartMetaData *chart.Metadata, refChartLocation string, templateName string, userId int32, pipelineStrategyPath string) (*ChartValues, error) GetChartVersion(location string) (string, error) - CreateChartProxy(chartMetaData *chart.Metadata, refChartLocation string, envName string, installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (string, *ChartGitAttribute, error) BuildChart(ctx context.Context, chartMetaData *chart.Metadata, referenceTemplatePath string) (string, error) BuildChartProxyForHelmApps(chartCreateRequest *ChartCreateRequest) (chartCreateResponse *ChartCreateResponse, err error) - GitPull(clonedDir string, repoUrl string, appStoreName string) error GetDir() string CleanDir(dir string) - GetUserEmailIdAndNameForGitOpsCommit(userId int32) (emailId, name string) - GetGitOpsRepoName(appName string) string - GetGitOpsRepoNameFromUrl(gitRepoUrl string) string - CreateGitRepositoryForApp(gitOpsRepoName, baseTemplateName, version string, userId int32) (chartGitAttribute *ChartGitAttribute, err error) - PushChartToGitRepo(gitOpsRepoName, referenceTemplate, version, tempReferenceTemplateDir string, repoUrl string, userId int32) (err error) GetByteArrayRefChart(chartMetaData *chart.Metadata, referenceTemplatePath string) ([]byte, error) - CreateReadmeInGitRepo(gitOpsRepoName string, userId int32) error - UpdateGitRepoUrlInCharts(appId int, chartGitAttribute *ChartGitAttribute, userId int32) error - CreateAndPushToGitChartProxy(appStoreName, tmpChartLocation string, envName string, installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (chartGitAttribute *ChartGitAttribute, err error) LoadChartInBytes(ChartPath string, deleteChart bool) ([]byte, error) LoadChartFromDir(dir string) (*chart.Chart, error) CreateZipFileForChart(chart *chart.Chart, outputChartPathDir string) ([]byte, error) + PackageChart(tempReferenceTemplateDir string, chartMetaData *chart.Metadata) (*string, string, error) } type ChartTemplateServiceImpl struct { - randSource rand.Source - logger *zap.SugaredLogger - gitFactory *GitFactory - globalEnvVariables *util.GlobalEnvVariables - gitOpsConfigRepository repository.GitOpsConfigRepository - userRepository repository2.UserRepository - chartRepository chartRepoRepository.ChartRepository + randSource rand.Source + logger *zap.SugaredLogger } type ChartValues struct { @@ -105,18 +82,10 @@ type ChartValues struct { ImageDescriptorTemplate string `json:"-"` } -func NewChartTemplateServiceImpl(logger *zap.SugaredLogger, - gitFactory *GitFactory, globalEnvVariables *util.GlobalEnvVariables, - gitOpsConfigRepository repository.GitOpsConfigRepository, - userRepository repository2.UserRepository, chartRepository chartRepoRepository.ChartRepository) *ChartTemplateServiceImpl { +func NewChartTemplateServiceImpl(logger *zap.SugaredLogger) *ChartTemplateServiceImpl { return &ChartTemplateServiceImpl{ - randSource: rand.NewSource(time.Now().UnixNano()), - logger: logger, - gitFactory: gitFactory, - globalEnvVariables: globalEnvVariables, - gitOpsConfigRepository: gitOpsConfigRepository, - userRepository: userRepository, - chartRepository: chartRepository, + randSource: rand.NewSource(time.Now().UnixNano()), + logger: logger, } } @@ -145,7 +114,7 @@ func (impl ChartTemplateServiceImpl) GetChartVersion(location string) (string, e return chartContent.Version, nil } -func (impl ChartTemplateServiceImpl) FetchValuesFromReferenceChart(chartMetaData *chart.Metadata, refChartLocation string, templateName string, userId int32, pipelineStrategyPath string) (*ChartValues, *ChartGitAttribute, error) { +func (impl ChartTemplateServiceImpl) FetchValuesFromReferenceChart(chartMetaData *chart.Metadata, refChartLocation string, templateName string, userId int32, pipelineStrategyPath string) (*ChartValues, error) { chartMetaData.ApiVersion = "v1" // ensure always v1 dir := impl.GetDir() chartDir := filepath.Join(ChartWorkingDirPath, dir) @@ -153,7 +122,7 @@ func (impl ChartTemplateServiceImpl) FetchValuesFromReferenceChart(chartMetaData err := os.MkdirAll(chartDir, os.ModePerm) //hack for concurrency handling if err != nil { impl.logger.Errorw("err in creating dir", "dir", chartDir, "err", err) - return nil, nil, err + return nil, err } defer impl.CleanDir(chartDir) @@ -161,27 +130,26 @@ func (impl ChartTemplateServiceImpl) FetchValuesFromReferenceChart(chartMetaData if err != nil { impl.logger.Errorw("error in copying chart for app", "app", chartMetaData.Name, "error", err) - return nil, nil, err + return nil, err } - archivePath, valuesYaml, err := impl.packageChart(chartDir, chartMetaData) + archivePath, valuesYaml, err := impl.PackageChart(chartDir, chartMetaData) if err != nil { impl.logger.Errorw("error in creating archive", "err", err) - return nil, nil, err + return nil, err } values, err := impl.getValues(chartDir, pipelineStrategyPath) if err != nil { impl.logger.Errorw("error in pushing chart", "path", archivePath, "err", err) - return nil, nil, err + return nil, err } values.Values = valuesYaml descriptor, err := ioutil.ReadFile(filepath.Clean(filepath.Join(chartDir, ".image_descriptor_template.json"))) if err != nil { impl.logger.Errorw("error in reading descriptor", "path", chartDir, "err", err) - return nil, nil, err + return nil, err } values.ImageDescriptorTemplate = string(descriptor) - chartGitAttr := &ChartGitAttribute{} - return values, chartGitAttr, nil + return values, nil } // TODO: convert BuildChart and BuildChartProxyForHelmApps into one function @@ -201,8 +169,8 @@ func (impl ChartTemplateServiceImpl) BuildChart(ctx context.Context, chartMetaDa impl.logger.Errorw("error in copying chart for app", "app", chartMetaData.Name, "error", err) return "", err } - _, span := otel.Tracer("orchestrator").Start(ctx, "impl.packageChart") - _, _, err = impl.packageChart(tempReferenceTemplateDir, chartMetaData) + _, span := otel.Tracer("orchestrator").Start(ctx, "impl.PackageChart") + _, _, err = impl.PackageChart(tempReferenceTemplateDir, chartMetaData) span.End() if err != nil { impl.logger.Errorw("error in creating archive", "err", err) @@ -229,7 +197,7 @@ func (impl ChartTemplateServiceImpl) BuildChartProxyForHelmApps(chartCreateReque impl.logger.Errorw("error in copying chart for app", "app", chartMetaData.Name, "error", err) return chartCreateResponse, err } - _, valuesYaml, err := impl.packageChart(chartDir, chartMetaData) + _, valuesYaml, err := impl.PackageChart(chartDir, chartMetaData) if err != nil { impl.logger.Errorw("error in creating archive", "err", err) return chartCreateResponse, err @@ -239,121 +207,6 @@ func (impl ChartTemplateServiceImpl) BuildChartProxyForHelmApps(chartCreateReque return chartCreateResponse, nil } -type ChartGitAttribute struct { - RepoUrl, ChartLocation string -} - -func (impl ChartTemplateServiceImpl) CreateGitRepositoryForApp(gitOpsRepoName, baseTemplateName, version string, userId int32) (chartGitAttribute *ChartGitAttribute, err error) { - //baseTemplateName replace whitespace - space := regexp.MustCompile(`\s+`) - gitOpsRepoName = space.ReplaceAllString(gitOpsRepoName, "-") - - gitOpsConfigBitbucket, err := impl.gitOpsConfigRepository.GetGitOpsConfigByProvider(BITBUCKET_PROVIDER) - if err != nil { - if err == pg.ErrNoRows { - gitOpsConfigBitbucket.BitBucketWorkspaceId = "" - } else { - return nil, err - } - } - //getting user name & emailId for commit author data - userEmailId, userName := impl.GetUserEmailIdAndNameForGitOpsCommit(userId) - gitRepoRequest := &bean.GitOpsConfigDto{ - GitRepoName: gitOpsRepoName, - Description: fmt.Sprintf("helm chart for " + gitOpsRepoName), - Username: userName, - UserEmailId: userEmailId, - BitBucketWorkspaceId: gitOpsConfigBitbucket.BitBucketWorkspaceId, - BitBucketProjectKey: gitOpsConfigBitbucket.BitBucketProjectKey, - } - repoUrl, _, detailedError := impl.gitFactory.Client.CreateRepository(gitRepoRequest) - for _, err := range detailedError.StageErrorMap { - if err != nil { - impl.logger.Errorw("error in creating git project", "name", gitOpsRepoName, "err", err) - return nil, err - } - } - return &ChartGitAttribute{RepoUrl: repoUrl, ChartLocation: filepath.Join(baseTemplateName, version)}, nil -} - -func (impl ChartTemplateServiceImpl) PushChartToGitRepo(gitOpsRepoName, referenceTemplate, version, tempReferenceTemplateDir string, repoUrl string, userId int32) (err error) { - chartDir := fmt.Sprintf("%s-%s", gitOpsRepoName, impl.GetDir()) - clonedDir := impl.gitFactory.GitService.GetCloneDirectory(chartDir) - if _, err := os.Stat(clonedDir); os.IsNotExist(err) { - clonedDir, err = impl.gitFactory.GitService.Clone(repoUrl, chartDir) - if err != nil { - impl.logger.Errorw("error in cloning repo", "url", repoUrl, "err", err) - return err - } - } else { - err = impl.GitPull(clonedDir, repoUrl, gitOpsRepoName) - if err != nil { - impl.logger.Errorw("error in pulling git repo", "url", repoUrl, "err", err) - return err - } - } - - dir := filepath.Join(clonedDir, referenceTemplate, version) - pushChartToGit := true - - //if chart already exists don't overrides it by reference template - if _, err := os.Stat(dir); os.IsNotExist(err) { - err = os.MkdirAll(dir, os.ModePerm) - if err != nil { - impl.logger.Errorw("error in making dir", "err", err) - return err - } - err = dirCopy.Copy(tempReferenceTemplateDir, dir) - if err != nil { - impl.logger.Errorw("error copying dir", "err", err) - return err - } - } else { - // auto-healing : data corruption fix - sometimes reference chart contents are not pushed in git-ops repo. - // copying content from reference template dir to cloned dir (if Chart.yaml file is not found) - // if Chart.yaml file is not found, we are assuming here that reference chart contents are not pushed in git-ops repo - if _, err := os.Stat(filepath.Join(dir, "Chart.yaml")); os.IsNotExist(err) { - impl.logger.Infow("auto-healing: Chart.yaml not found in cloned repo from git-ops. copying content", "from", tempReferenceTemplateDir, "to", dir) - err = dirCopy.Copy(tempReferenceTemplateDir, dir) - if err != nil { - impl.logger.Errorw("error copying content in auto-healing", "err", err) - return err - } - } else { - // chart exists on git, hence not performing first commit - pushChartToGit = false - } - } - - // if push needed, then only push - if pushChartToGit { - userEmailId, userName := impl.GetUserEmailIdAndNameForGitOpsCommit(userId) - commit, err := impl.gitFactory.GitService.CommitAndPushAllChanges(clonedDir, "first commit", userName, userEmailId) - if err != nil { - impl.logger.Errorw("error in pushing git", "err", err) - impl.logger.Warn("re-trying, taking pull and then push again") - err = impl.GitPull(clonedDir, repoUrl, gitOpsRepoName) - if err != nil { - return err - } - err = dirCopy.Copy(tempReferenceTemplateDir, dir) - if err != nil { - impl.logger.Errorw("error copying dir", "err", err) - return err - } - commit, err = impl.gitFactory.GitService.CommitAndPushAllChanges(clonedDir, "first commit", userName, userEmailId) - if err != nil { - impl.logger.Errorw("error in pushing git", "err", err) - return err - } - } - impl.logger.Debugw("template committed", "url", repoUrl, "commit", commit) - } - - defer impl.CleanDir(clonedDir) - return nil -} - func (impl ChartTemplateServiceImpl) getValues(directory, pipelineStrategyPath string) (values *ChartValues, err error) { if fi, err := os.Stat(directory); err != nil { @@ -428,7 +281,7 @@ func (impl ChartTemplateServiceImpl) getValues(directory, pipelineStrategyPath s } -func (impl ChartTemplateServiceImpl) packageChart(tempReferenceTemplateDir string, chartMetaData *chart.Metadata) (*string, string, error) { +func (impl ChartTemplateServiceImpl) PackageChart(tempReferenceTemplateDir string, chartMetaData *chart.Metadata) (*string, string, error) { valid, err := chartutil.IsChartDir(tempReferenceTemplateDir) if err != nil { impl.logger.Errorw("error in validating base chart", "dir", tempReferenceTemplateDir, "err", err) @@ -479,187 +332,6 @@ func (impl ChartTemplateServiceImpl) GetDir() string { return strconv.FormatInt(r1, 10) } -func (impl ChartTemplateServiceImpl) CreateChartProxy(chartMetaData *chart.Metadata, refChartLocation string, envName string, installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (string, *ChartGitAttribute, error) { - chartMetaData.ApiVersion = "v2" // ensure always v2 - dir := impl.GetDir() - chartDir := filepath.Join(ChartWorkingDirPath, dir) - impl.logger.Debugw("chart dir ", "chart", chartMetaData.Name, "dir", chartDir) - err := os.MkdirAll(chartDir, os.ModePerm) //hack for concurrency handling - if err != nil { - impl.logger.Errorw("err in creating dir", "dir", chartDir, "err", err) - return "", nil, err - } - defer impl.CleanDir(chartDir) - err = dirCopy.Copy(refChartLocation, chartDir) - - if err != nil { - impl.logger.Errorw("error in copying chart for app", "app", chartMetaData.Name, "error", err) - return "", nil, err - } - archivePath, valuesYaml, err := impl.packageChart(chartDir, chartMetaData) - if err != nil { - impl.logger.Errorw("error in creating archive", "err", err) - return "", nil, err - } - - chartGitAttr, err := impl.CreateAndPushToGitChartProxy(chartMetaData.Name, chartDir, envName, installAppVersionRequest) - if err != nil { - impl.logger.Errorw("error in pushing chart to git ", "path", archivePath, "err", err) - return "", nil, err - } - if valuesYaml == "" { - valuesYaml = "{}" - } else { - valuesYamlByte, err := yaml.YAMLToJSON([]byte(valuesYaml)) - if err != nil { - return "", nil, err - } - valuesYaml = string(valuesYamlByte) - } - return valuesYaml, chartGitAttr, nil -} - -func (impl ChartTemplateServiceImpl) CreateAndPushToGitChartProxy(appStoreName, tmpChartLocation string, envName string, installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (chartGitAttribute *ChartGitAttribute, err error) { - //baseTemplateName replace whitespace - space := regexp.MustCompile(`\s+`) - appStoreName = space.ReplaceAllString(appStoreName, "-") - - if len(installAppVersionRequest.GitOpsRepoName) == 0 { - //here git ops repo will be the app name, to breaking the mono repo structure - gitOpsRepoName := impl.GetGitOpsRepoName(installAppVersionRequest.AppName) - installAppVersionRequest.GitOpsRepoName = gitOpsRepoName - } - gitOpsConfigBitbucket, err := impl.gitOpsConfigRepository.GetGitOpsConfigByProvider(BITBUCKET_PROVIDER) - if err != nil { - if err == pg.ErrNoRows { - gitOpsConfigBitbucket.BitBucketWorkspaceId = "" - } else { - return nil, err - } - } - //getting user name & emailId for commit author data - userEmailId, userName := impl.GetUserEmailIdAndNameForGitOpsCommit(installAppVersionRequest.UserId) - gitRepoRequest := &bean.GitOpsConfigDto{ - GitRepoName: installAppVersionRequest.GitOpsRepoName, - Description: "helm chart for " + installAppVersionRequest.GitOpsRepoName, - Username: userName, - UserEmailId: userEmailId, - BitBucketWorkspaceId: gitOpsConfigBitbucket.BitBucketWorkspaceId, - BitBucketProjectKey: gitOpsConfigBitbucket.BitBucketProjectKey, - } - repoUrl, _, detailedError := impl.gitFactory.Client.CreateRepository(gitRepoRequest) - for _, err := range detailedError.StageErrorMap { - if err != nil { - impl.logger.Errorw("error in creating git project", "name", installAppVersionRequest.GitOpsRepoName, "err", err) - return nil, err - } - } - - chartDir := fmt.Sprintf("%s-%s", installAppVersionRequest.AppName, impl.GetDir()) - clonedDir := impl.gitFactory.GitService.GetCloneDirectory(chartDir) - if _, err := os.Stat(clonedDir); os.IsNotExist(err) { - clonedDir, err = impl.gitFactory.GitService.Clone(repoUrl, chartDir) - if err != nil { - impl.logger.Errorw("error in cloning repo", "url", repoUrl, "err", err) - return nil, err - } - } else { - err = impl.GitPull(clonedDir, repoUrl, appStoreName) - if err != nil { - return nil, err - } - } - - acdAppName := fmt.Sprintf("%s-%s", installAppVersionRequest.AppName, envName) - dir := filepath.Join(clonedDir, acdAppName) - err = os.MkdirAll(dir, os.ModePerm) - if err != nil { - impl.logger.Errorw("error in making dir", "err", err) - return nil, err - } - err = dirCopy.Copy(tmpChartLocation, dir) - if err != nil { - impl.logger.Errorw("error copying dir", "err", err) - return nil, err - } - commit, err := impl.gitFactory.GitService.CommitAndPushAllChanges(clonedDir, "first commit", userName, userEmailId) - if err != nil { - impl.logger.Errorw("error in pushing git", "err", err) - impl.logger.Warn("re-trying, taking pull and then push again") - err = impl.GitPull(clonedDir, repoUrl, acdAppName) - if err != nil { - return nil, err - } - err = dirCopy.Copy(tmpChartLocation, dir) - if err != nil { - impl.logger.Errorw("error copying dir", "err", err) - return nil, err - } - commit, err = impl.gitFactory.GitService.CommitAndPushAllChanges(clonedDir, "first commit", userName, userEmailId) - if err != nil { - impl.logger.Errorw("error in pushing git", "err", err) - return nil, err - } - } - impl.logger.Debugw("template committed", "url", repoUrl, "commit", commit) - defer impl.CleanDir(clonedDir) - return &ChartGitAttribute{RepoUrl: repoUrl, ChartLocation: filepath.Join("", acdAppName)}, nil -} - -func (impl ChartTemplateServiceImpl) GitPull(clonedDir string, repoUrl string, appStoreName string) error { - err := impl.gitFactory.GitService.Pull(clonedDir) //TODO check for local repo exists before clone - if err != nil { - impl.logger.Errorw("error in pulling git", "clonedDir", clonedDir, "err", err) - _, err := impl.gitFactory.GitService.Clone(repoUrl, appStoreName) - if err != nil { - impl.logger.Errorw("error in cloning repo", "url", repoUrl, "err", err) - return err - } - return nil - } - return nil -} - -func (impl *ChartTemplateServiceImpl) GetUserEmailIdAndNameForGitOpsCommit(userId int32) (string, string) { - emailId := "devtron-bot@devtron.ai" - name := "devtron bot" - //getting emailId associated with user - userDetail, _ := impl.userRepository.GetById(userId) - if userDetail != nil && userDetail.EmailId != "admin" && userDetail.EmailId != "system" && len(userDetail.EmailId) > 0 { - emailId = userDetail.EmailId - } else { - emailIdGitOps, err := impl.gitOpsConfigRepository.GetEmailIdFromActiveGitOpsConfig() - if err != nil { - impl.logger.Errorw("error in getting emailId from active gitOps config", "err", err) - } else if len(emailIdGitOps) > 0 { - emailId = emailIdGitOps - } - } - //we are getting name from emailId(replacing special characters in with space) - emailComponents := strings.Split(emailId, "@") - regex, _ := regexp.Compile(`[^\w]`) - if regex != nil { - name = regex.ReplaceAllString(emailComponents[0], " ") - } - return emailId, name -} - -func (impl ChartTemplateServiceImpl) GetGitOpsRepoName(appName string) string { - var repoName string - if len(impl.globalEnvVariables.GitOpsRepoPrefix) == 0 { - repoName = appName - } else { - repoName = fmt.Sprintf("%s-%s", impl.globalEnvVariables.GitOpsRepoPrefix, appName) - } - return repoName -} - -func (impl ChartTemplateServiceImpl) GetGitOpsRepoNameFromUrl(gitRepoUrl string) string { - gitRepoUrl = gitRepoUrl[strings.LastIndex(gitRepoUrl, "/")+1:] - gitRepoUrl = strings.ReplaceAll(gitRepoUrl, ".git", "") - return gitRepoUrl -} - // GetByteArrayRefChart this method will be used for getting byte array from reference chart to store in db func (impl ChartTemplateServiceImpl) GetByteArrayRefChart(chartMetaData *chart.Metadata, referenceTemplatePath string) ([]byte, error) { chartMetaData.ApiVersion = "v1" // ensure always v1 @@ -677,7 +349,7 @@ func (impl ChartTemplateServiceImpl) GetByteArrayRefChart(chartMetaData *chart.M impl.logger.Errorw("error in copying chart for app", "app", chartMetaData.Name, "error", err) return nil, err } - activePath, _, err := impl.packageChart(tempReferenceTemplateDir, chartMetaData) + activePath, _, err := impl.PackageChart(tempReferenceTemplateDir, chartMetaData) if err != nil { impl.logger.Errorw("error in creating archive", "err", err) return nil, err @@ -697,42 +369,6 @@ func (impl ChartTemplateServiceImpl) GetByteArrayRefChart(chartMetaData *chart.M return bs, nil } -func (impl ChartTemplateServiceImpl) CreateReadmeInGitRepo(gitOpsRepoName string, userId int32) error { - userEmailId, userName := impl.GetUserEmailIdAndNameForGitOpsCommit(userId) - gitOpsConfig, err := impl.gitOpsConfigRepository.GetGitOpsConfigActive() - config := &bean.GitOpsConfigDto{ - Username: userName, - UserEmailId: userEmailId, - GitRepoName: gitOpsRepoName, - BitBucketWorkspaceId: gitOpsConfig.BitBucketWorkspaceId, - } - _, err = impl.gitFactory.Client.CreateReadme(config) - if err != nil { - return err - } - return nil -} - -func (impl ChartTemplateServiceImpl) UpdateGitRepoUrlInCharts(appId int, chartGitAttribute *ChartGitAttribute, userId int32) error { - charts, err := impl.chartRepository.FindActiveChartsByAppId(appId) - if err != nil && pg.ErrNoRows != err { - return err - } - for _, ch := range charts { - if len(ch.GitRepoUrl) == 0 { - ch.GitRepoUrl = chartGitAttribute.RepoUrl - ch.ChartLocation = chartGitAttribute.ChartLocation - ch.UpdatedOn = time.Now() - ch.UpdatedBy = userId - err = impl.chartRepository.Update(ch) - if err != nil { - return err - } - } - } - return nil -} - func (impl ChartTemplateServiceImpl) LoadChartInBytes(ChartPath string, deleteChart bool) ([]byte, error) { var chartBytesArr []byte diff --git a/internal/util/GitService.go b/internal/util/GitService.go index 882772e523..06b2a47d63 100644 --- a/internal/util/GitService.go +++ b/internal/util/GitService.go @@ -52,6 +52,7 @@ const ( ) type GitClient interface { + //TODO: make all methods independent of GitOpsConfigDto, create own dto object CreateRepository(config *bean2.GitOpsConfigDto) (url string, isNew bool, detailedErrorGitOpsConfigActions DetailedErrorGitOpsConfigActions) CommitValues(config *ChartConfig, gitOpsConfig *bean2.GitOpsConfigDto) (commitHash string, commitTime time.Time, err error) GetRepoUrl(config *bean2.GitOpsConfigDto) (repoUrl string, err error) diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index 5a17c7dfcd..94898950cf 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -21,6 +21,9 @@ import ( "context" "encoding/json" "fmt" + commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" "io/ioutil" "net/url" @@ -117,6 +120,8 @@ type AppServiceImpl struct { scopedVariableManager variables.ScopedVariableCMCSManager acdConfig *argocdServer.ACDConfig chartRefService chartRef.ChartRefService + gitOpsConfigReadService config.GitOpsConfigReadService + gitOpsRemoteOperationService remote.GitOpsRemoteOperationService } type AppService interface { @@ -136,7 +141,7 @@ type AppService interface { //GetEnvOverrideByTriggerType(overrideRequest *bean.ValuesOverrideRequest, triggeredAt time.Time, ctx context.Context) (*chartConfig.EnvConfigOverride, error) //GetAppMetricsByTriggerType(overrideRequest *bean.ValuesOverrideRequest, ctx context.Context) (bool, error) //GetDeploymentStrategyByTriggerType(overrideRequest *bean.ValuesOverrideRequest, ctx context.Context) (*chartConfig.PipelineStrategy, error) - CreateGitopsRepo(app *app.App, userId int32) (gitopsRepoName string, chartGitAttr *ChartGitAttribute, err error) + CreateGitopsRepo(app *app.App, userId int32) (gitopsRepoName string, chartGitAttr *commonBean.ChartGitAttribute, err error) GetDeployedManifestByPipelineIdAndCDWorkflowId(appId int, envId int, cdWorkflowId int, ctx context.Context) ([]byte, error) //SetPipelineFieldsInOverrideRequest(overrideRequest *bean.ValuesOverrideRequest, pipeline *pipelineConfig.Pipeline) @@ -169,7 +174,9 @@ func NewAppService( installedAppVersionHistoryRepository repository4.InstalledAppVersionHistoryRepository, globalEnvVariables *util2.GlobalEnvVariables, scopedVariableManager variables.ScopedVariableCMCSManager, - acdConfig *argocdServer.ACDConfig, chartRefService chartRef.ChartRefService) *AppServiceImpl { + acdConfig *argocdServer.ACDConfig, chartRefService chartRef.ChartRefService, + gitOpsConfigReadService config.GitOpsConfigReadService, + gitOpsRemoteOperationService remote.GitOpsRemoteOperationService) *AppServiceImpl { appServiceImpl := &AppServiceImpl{ environmentConfigRepository: environmentConfigRepository, mergeUtil: mergeUtil, @@ -198,6 +205,8 @@ func NewAppService( scopedVariableManager: scopedVariableManager, acdConfig: acdConfig, chartRefService: chartRefService, + gitOpsConfigReadService: gitOpsConfigReadService, + gitOpsRemoteOperationService: gitOpsRemoteOperationService, } return appServiceImpl } @@ -901,13 +910,13 @@ func (impl *AppServiceImpl) BuildChartAndGetPath(appName string, envOverride *ch return tempReferenceTemplateDir, nil } -func (impl *AppServiceImpl) CreateGitopsRepo(app *app.App, userId int32) (gitopsRepoName string, chartGitAttr *ChartGitAttribute, err error) { +func (impl *AppServiceImpl) CreateGitopsRepo(app *app.App, userId int32) (gitopsRepoName string, chartGitAttr *commonBean.ChartGitAttribute, err error) { chart, err := impl.chartRepository.FindLatestChartForAppByAppId(app.Id) if err != nil && pg.ErrNoRows != err { return "", nil, err } - gitOpsRepoName := impl.chartTemplateService.GetGitOpsRepoName(app.AppName) - chartGitAttr, err = impl.chartTemplateService.CreateGitRepositoryForApp(gitOpsRepoName, chart.ReferenceTemplate, chart.ChartVersion, userId) + gitOpsRepoName := impl.gitOpsConfigReadService.GetGitOpsRepoName(app.AppName) + chartGitAttr, err = impl.gitOpsRemoteOperationService.CreateGitRepositoryForApp(gitOpsRepoName, chart.ReferenceTemplate, chart.ChartVersion, userId) if err != nil { impl.logger.Errorw("error in pushing chart to git ", "gitOpsRepoName", gitOpsRepoName, "err", err) return "", nil, err diff --git a/pkg/app/ManifestPushService.go b/pkg/app/ManifestPushService.go index f52731b204..1e32701e53 100644 --- a/pkg/app/ManifestPushService.go +++ b/pkg/app/ManifestPushService.go @@ -5,14 +5,14 @@ import ( "fmt" bean2 "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/client/argocdServer" - "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" . "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app/bean" status2 "github.com/devtron-labs/devtron/pkg/app/status" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" - "github.com/go-pg/pg" "go.opentelemetry.io/otel" "go.uber.org/zap" "time" @@ -28,32 +28,32 @@ type GitOpsPushService interface { type GitOpsManifestPushServiceImpl struct { logger *zap.SugaredLogger - chartTemplateService util.ChartTemplateService - gitOpsConfigRepository repository.GitOpsConfigRepository gitFactory *GitFactory pipelineStatusTimelineService status2.PipelineStatusTimelineService pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository acdConfig *argocdServer.ACDConfig chartRefService chartRef.ChartRefService + gitOpsConfigReadService config.GitOpsConfigReadService + gitOpsRemoteOperationService remote.GitOpsRemoteOperationService } func NewGitOpsManifestPushServiceImpl( logger *zap.SugaredLogger, - chartTemplateService util.ChartTemplateService, - gitOpsConfigRepository repository.GitOpsConfigRepository, gitFactory *GitFactory, pipelineStatusTimelineService status2.PipelineStatusTimelineService, pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository, - acdConfig *argocdServer.ACDConfig, chartRefService chartRef.ChartRefService) *GitOpsManifestPushServiceImpl { + acdConfig *argocdServer.ACDConfig, chartRefService chartRef.ChartRefService, + gitOpsConfigReadService config.GitOpsConfigReadService, + gitOpsRemoteOperationService remote.GitOpsRemoteOperationService) *GitOpsManifestPushServiceImpl { return &GitOpsManifestPushServiceImpl{ logger: logger, - chartTemplateService: chartTemplateService, - gitOpsConfigRepository: gitOpsConfigRepository, gitFactory: gitFactory, pipelineStatusTimelineService: pipelineStatusTimelineService, pipelineStatusTimelineRepository: pipelineStatusTimelineRepository, acdConfig: acdConfig, chartRefService: chartRefService, + gitOpsConfigReadService: gitOpsConfigReadService, + gitOpsRemoteOperationService: gitOpsRemoteOperationService, } } @@ -105,7 +105,7 @@ func (impl *GitOpsManifestPushServiceImpl) PushChartToGitRepo(manifestPushTempla _, span := otel.Tracer("orchestrator").Start(ctx, "chartTemplateService.GetGitOpsRepoName") // CHART COMMIT and PUSH STARTS HERE, it will push latest version, if found modified on deployment template and overrides - gitOpsRepoName := impl.chartTemplateService.GetGitOpsRepoName(manifestPushTemplate.AppName) + gitOpsRepoName := impl.gitOpsConfigReadService.GetGitOpsRepoName(manifestPushTemplate.AppName) span.End() _, span = otel.Tracer("orchestrator").Start(ctx, "chartService.CheckChartExists") err := impl.chartRefService.CheckChartExists(manifestPushTemplate.ChartRefId) @@ -114,7 +114,7 @@ func (impl *GitOpsManifestPushServiceImpl) PushChartToGitRepo(manifestPushTempla impl.logger.Errorw("err in getting chart info", "err", err) return err } - err = impl.chartTemplateService.PushChartToGitRepo(gitOpsRepoName, manifestPushTemplate.ChartReferenceTemplate, manifestPushTemplate.ChartVersion, manifestPushTemplate.BuiltChartPath, manifestPushTemplate.RepoUrl, manifestPushTemplate.UserId) + err = impl.gitOpsRemoteOperationService.PushChartToGitRepo(gitOpsRepoName, manifestPushTemplate.ChartReferenceTemplate, manifestPushTemplate.ChartVersion, manifestPushTemplate.BuiltChartPath, manifestPushTemplate.RepoUrl, manifestPushTemplate.UserId) if err != nil { impl.logger.Errorw("error in pushing chart to git", "err", err) return err @@ -125,10 +125,10 @@ func (impl *GitOpsManifestPushServiceImpl) PushChartToGitRepo(manifestPushTempla func (impl *GitOpsManifestPushServiceImpl) CommitValuesToGit(manifestPushTemplate *bean.ManifestPushTemplate, ctx context.Context) (commitHash string, commitTime time.Time, err error) { commitHash = "" commitTime = time.Time{} - chartRepoName := impl.chartTemplateService.GetGitOpsRepoNameFromUrl(manifestPushTemplate.RepoUrl) + chartRepoName := impl.gitOpsConfigReadService.GetGitOpsRepoNameFromUrl(manifestPushTemplate.RepoUrl) _, span := otel.Tracer("orchestrator").Start(ctx, "chartTemplateService.GetUserEmailIdAndNameForGitOpsCommit") //getting username & emailId for commit author data - userEmailId, userName := impl.chartTemplateService.GetUserEmailIdAndNameForGitOpsCommit(manifestPushTemplate.UserId) + userEmailId, userName := impl.gitOpsConfigReadService.GetUserEmailIdAndNameForGitOpsCommit(manifestPushTemplate.UserId) span.End() chartGitAttr := &util.ChartConfig{ FileName: fmt.Sprintf("_%d-values.yaml", manifestPushTemplate.TargetEnvironmentName), @@ -140,15 +140,12 @@ func (impl *GitOpsManifestPushServiceImpl) CommitValuesToGit(manifestPushTemplat UserName: userName, UserEmailId: userEmailId, } - gitOpsConfigBitbucket, err := impl.gitOpsConfigRepository.GetGitOpsConfigByProvider(util.BITBUCKET_PROVIDER) + bitbucketMetadata, err := impl.gitOpsConfigReadService.GetBitbucketMetadata() if err != nil { - if err == pg.ErrNoRows { - gitOpsConfigBitbucket.BitBucketWorkspaceId = "" - } else { - return commitHash, commitTime, err - } + impl.logger.Errorw("error in getting bitbucket metadata", "err", err) + return commitHash, commitTime, err } - gitOpsConfig := &bean2.GitOpsConfigDto{BitBucketWorkspaceId: gitOpsConfigBitbucket.BitBucketWorkspaceId} + gitOpsConfig := &bean2.GitOpsConfigDto{BitBucketWorkspaceId: bitbucketMetadata.BitBucketWorkspaceId} _, span = otel.Tracer("orchestrator").Start(ctx, "gitFactory.Client.CommitValues") commitHash, commitTime, err = impl.gitFactory.Client.CommitValues(chartGitAttr, gitOpsConfig) span.End() diff --git a/pkg/appClone/AppCloneService.go b/pkg/appClone/AppCloneService.go index 68fac6ef78..8affc7f2e2 100644 --- a/pkg/appClone/AppCloneService.go +++ b/pkg/appClone/AppCloneService.go @@ -31,6 +31,7 @@ import ( "github.com/devtron-labs/devtron/pkg/appWorkflow" "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/chart" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" "github.com/devtron-labs/devtron/pkg/pipeline" bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/go-pg/pg" @@ -44,7 +45,6 @@ type AppCloneService interface { type AppCloneServiceImpl struct { logger *zap.SugaredLogger pipelineBuilder pipeline.PipelineBuilder - materialRepository pipelineConfig.MaterialRepository chartService chart.ChartService configMapService pipeline.ConfigMapService appWorkflowService appWorkflow.AppWorkflowService @@ -55,27 +55,25 @@ type AppCloneServiceImpl struct { appRepository app2.AppRepository ciPipelineRepository pipelineConfig.CiPipelineRepository pipelineRepository pipelineConfig.PipelineRepository - appWorkflowRepository appWorkflow2.AppWorkflowRepository ciPipelineConfigService pipeline.CiPipelineConfigService + gitOpsConfigReadService config.GitOpsConfigReadService } func NewAppCloneServiceImpl(logger *zap.SugaredLogger, pipelineBuilder pipeline.PipelineBuilder, - materialRepository pipelineConfig.MaterialRepository, chartService chart.ChartService, configMapService pipeline.ConfigMapService, appWorkflowService appWorkflow.AppWorkflowService, appListingService app.AppListingService, propertiesConfigService pipeline.PropertiesConfigService, - ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository, pipelineStageService pipeline.PipelineStageService, ciTemplateService pipeline.CiTemplateService, appRepository app2.AppRepository, ciPipelineRepository pipelineConfig.CiPipelineRepository, - pipelineRepository pipelineConfig.PipelineRepository, appWorkflowRepository appWorkflow2.AppWorkflowRepository, - ciPipelineConfigService pipeline.CiPipelineConfigService) *AppCloneServiceImpl { + pipelineRepository pipelineConfig.PipelineRepository, + ciPipelineConfigService pipeline.CiPipelineConfigService, + gitOpsConfigReadService config.GitOpsConfigReadService) *AppCloneServiceImpl { return &AppCloneServiceImpl{ logger: logger, pipelineBuilder: pipelineBuilder, - materialRepository: materialRepository, chartService: chartService, configMapService: configMapService, appWorkflowService: appWorkflowService, @@ -86,8 +84,8 @@ func NewAppCloneServiceImpl(logger *zap.SugaredLogger, appRepository: appRepository, ciPipelineRepository: ciPipelineRepository, pipelineRepository: pipelineRepository, - appWorkflowRepository: appWorkflowRepository, ciPipelineConfigService: ciPipelineConfigService, + gitOpsConfigReadService: gitOpsConfigReadService, } } @@ -985,7 +983,7 @@ func (impl *AppCloneServiceImpl) CreateCdPipeline(req *cloneCdPipelineRequest, c for deploymentType, allowed := range DeploymentAppConfigForEnvironment { AllowedDeploymentAppTypes[deploymentType] = allowed } - isGitopsConfigured, err := impl.pipelineBuilder.IsGitopsConfigured() + isGitopsConfigured, err := impl.gitOpsConfigReadService.IsGitOpsConfigured() if err != nil { impl.logger.Errorw("error in checking if gitOps configured", "err", err) return nil, err diff --git a/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go b/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go index ab912fda04..ed86999b8b 100644 --- a/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go +++ b/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go @@ -21,13 +21,15 @@ import ( "encoding/json" "fmt" "github.com/devtron-labs/devtron/api/bean" - repository3 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/util" appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" appStoreDiscoverRepository "github.com/devtron-labs/devtron/pkg/appStore/discover/repository" util2 "github.com/devtron-labs/devtron/pkg/appStore/util" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" + commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote" "github.com/go-pg/pg" "github.com/google/go-github/github" "github.com/microsoft/azure-devops-go-api/azuredevops" @@ -50,7 +52,7 @@ type AppStoreDeploymentCommonService interface { ParseGitRepoErrorResponse(err error) (bool, error) GetValuesAndRequirementGitConfig(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*util.ChartConfig, *util.ChartConfig, error) CreateChartProxyAndGetPath(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*util.ChartCreateResponse, error) - CreateGitOpsRepoAndPushChart(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, builtChartPath string, requirementsConfig *util.ChartConfig, valuesConfig *util.ChartConfig) (*util.ChartGitAttribute, bool, string, error) + CreateGitOpsRepoAndPushChart(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, builtChartPath string, requirementsConfig *util.ChartConfig, valuesConfig *util.ChartConfig) (*commonBean.ChartGitAttribute, bool, string, error) CommitConfigToGit(chartConfig *util.ChartConfig) (gitHash string, err error) GetGitCommitConfig(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, fileString string, filename string) (*util.ChartConfig, error) GetValuesString(chartName, valuesOverrideYaml string) (string, error) @@ -67,7 +69,8 @@ type AppStoreDeploymentCommonServiceImpl struct { environmentRepository repository2.EnvironmentRepository chartTemplateService util.ChartTemplateService gitFactory *util.GitFactory - gitOpsConfigRepository repository3.GitOpsConfigRepository + gitOpsConfigReadService config.GitOpsConfigReadService + gitOpsRemoteOperationService remote.GitOpsRemoteOperationService } func NewAppStoreDeploymentCommonServiceImpl( @@ -77,8 +80,8 @@ func NewAppStoreDeploymentCommonServiceImpl( environmentRepository repository2.EnvironmentRepository, chartTemplateService util.ChartTemplateService, gitFactory *util.GitFactory, - gitOpsConfigRepository repository3.GitOpsConfigRepository, -) *AppStoreDeploymentCommonServiceImpl { + gitOpsConfigReadService config.GitOpsConfigReadService, + gitOpsRemoteOperationService remote.GitOpsRemoteOperationService) *AppStoreDeploymentCommonServiceImpl { return &AppStoreDeploymentCommonServiceImpl{ logger: logger, installedAppRepository: installedAppRepository, @@ -86,7 +89,8 @@ func NewAppStoreDeploymentCommonServiceImpl( environmentRepository: environmentRepository, chartTemplateService: chartTemplateService, gitFactory: gitFactory, - gitOpsConfigRepository: gitOpsConfigRepository, + gitOpsConfigReadService: gitOpsConfigReadService, + gitOpsRemoteOperationService: gitOpsRemoteOperationService, } } @@ -117,7 +121,7 @@ type AppStoreManifestResponse struct { } type AppStoreGitOpsResponse struct { - ChartGitAttribute *util.ChartGitAttribute + ChartGitAttribute *commonBean.ChartGitAttribute GitHash string } @@ -299,8 +303,8 @@ func (impl AppStoreDeploymentCommonServiceImpl) GetGitCommitConfig(installAppVer } argocdAppName := installAppVersionRequest.AppName + "-" + environment.Name - gitOpsRepoName := impl.chartTemplateService.GetGitOpsRepoName(installAppVersionRequest.AppName) - userEmailId, userName := impl.chartTemplateService.GetUserEmailIdAndNameForGitOpsCommit(installAppVersionRequest.UserId) + gitOpsRepoName := impl.gitOpsConfigReadService.GetGitOpsRepoName(installAppVersionRequest.AppName) + userEmailId, userName := impl.gitOpsConfigReadService.GetUserEmailIdAndNameForGitOpsCommit(installAppVersionRequest.UserId) YamlConfig := &util.ChartConfig{ FileName: filename, FileContent: fileString, @@ -403,27 +407,23 @@ func (impl AppStoreDeploymentCommonServiceImpl) CreateGitOpsRepo(installAppVersi if len(installAppVersionRequest.GitOpsRepoName) == 0 { //here gitops repo will be the app name, to breaking the mono repo structure - gitOpsRepoName := impl.chartTemplateService.GetGitOpsRepoName(installAppVersionRequest.AppName) + gitOpsRepoName := impl.gitOpsConfigReadService.GetGitOpsRepoName(installAppVersionRequest.AppName) installAppVersionRequest.GitOpsRepoName = gitOpsRepoName } - gitOpsConfigBitbucket, err := impl.gitOpsConfigRepository.GetGitOpsConfigByProvider(util.BITBUCKET_PROVIDER) + bitbucketMetadata, err := impl.gitOpsConfigReadService.GetBitbucketMetadata() if err != nil { - if err == pg.ErrNoRows { - err = nil - gitOpsConfigBitbucket.BitBucketWorkspaceId = "" - } else { - return "", false, err - } + impl.logger.Errorw("error in getting bitbucket metadata", "err", err) + return "", false, err } //getting user name & emailId for commit author data - userEmailId, userName := impl.chartTemplateService.GetUserEmailIdAndNameForGitOpsCommit(installAppVersionRequest.UserId) + userEmailId, userName := impl.gitOpsConfigReadService.GetUserEmailIdAndNameForGitOpsCommit(installAppVersionRequest.UserId) gitRepoRequest := &bean.GitOpsConfigDto{ GitRepoName: installAppVersionRequest.GitOpsRepoName, Description: "helm chart for " + installAppVersionRequest.GitOpsRepoName, Username: userName, UserEmailId: userEmailId, - BitBucketWorkspaceId: gitOpsConfigBitbucket.BitBucketWorkspaceId, - BitBucketProjectKey: gitOpsConfigBitbucket.BitBucketProjectKey, + BitBucketWorkspaceId: bitbucketMetadata.BitBucketWorkspaceId, + BitBucketProjectKey: bitbucketMetadata.BitBucketProjectKey, } repoUrl, isNew, detailedError := impl.gitFactory.Client.CreateRepository(gitRepoRequest) for _, err := range detailedError.StageErrorMap { @@ -436,7 +436,7 @@ func (impl AppStoreDeploymentCommonServiceImpl) CreateGitOpsRepo(installAppVersi } // PushChartToGitopsRepo pushes built chart to gitOps repo -func (impl AppStoreDeploymentCommonServiceImpl) PushChartToGitopsRepo(PushChartToGitRequest *appStoreBean.PushChartToGitRequestDTO, requirementsConfig *util.ChartConfig, valuesConfig *util.ChartConfig) (*util.ChartGitAttribute, string, error) { +func (impl AppStoreDeploymentCommonServiceImpl) PushChartToGitopsRepo(PushChartToGitRequest *appStoreBean.PushChartToGitRequestDTO, requirementsConfig *util.ChartConfig, valuesConfig *util.ChartConfig) (*commonBean.ChartGitAttribute, string, error) { space := regexp.MustCompile(`\s+`) appStoreName := space.ReplaceAllString(PushChartToGitRequest.ChartAppStoreName, "-") chartDir := fmt.Sprintf("%s-%s", PushChartToGitRequest.AppName, impl.chartTemplateService.GetDir()) @@ -448,7 +448,7 @@ func (impl AppStoreDeploymentCommonServiceImpl) PushChartToGitopsRepo(PushChartT return nil, "", err } } else { - err = impl.chartTemplateService.GitPull(clonedDir, PushChartToGitRequest.RepoURL, appStoreName) + err = impl.gitOpsRemoteOperationService.GitPull(clonedDir, PushChartToGitRequest.RepoURL, appStoreName) if err != nil { return nil, "", err } @@ -475,12 +475,12 @@ func (impl AppStoreDeploymentCommonServiceImpl) PushChartToGitopsRepo(PushChartT impl.logger.Errorw("error in adding values.yaml to chart", "err", err, "appName", PushChartToGitRequest.AppName) return nil, "", err } - userEmailId, userName := impl.chartTemplateService.GetUserEmailIdAndNameForGitOpsCommit(PushChartToGitRequest.UserId) + userEmailId, userName := impl.gitOpsConfigReadService.GetUserEmailIdAndNameForGitOpsCommit(PushChartToGitRequest.UserId) commit, err := impl.gitFactory.GitService.CommitAndPushAllChanges(clonedDir, "first commit", userName, userEmailId) if err != nil { impl.logger.Errorw("error in pushing git", "err", err) impl.logger.Warn("re-trying, taking pull and then push again") - err = impl.chartTemplateService.GitPull(clonedDir, PushChartToGitRequest.RepoURL, acdAppName) + err = impl.gitOpsRemoteOperationService.GitPull(clonedDir, PushChartToGitRequest.RepoURL, acdAppName) if err != nil { impl.logger.Errorw("error in git pull", "err", err, "appName", acdAppName) return nil, "", err @@ -498,7 +498,7 @@ func (impl AppStoreDeploymentCommonServiceImpl) PushChartToGitopsRepo(PushChartT } impl.logger.Debugw("template committed", "url", PushChartToGitRequest.RepoURL, "commit", commit) defer impl.chartTemplateService.CleanDir(clonedDir) - return &util.ChartGitAttribute{RepoUrl: PushChartToGitRequest.RepoURL, ChartLocation: acdAppName}, commit, err + return &commonBean.ChartGitAttribute{RepoUrl: PushChartToGitRequest.RepoURL, ChartLocation: acdAppName}, commit, err } // AddConfigFileToChart will override requirements.yaml file in chart @@ -525,7 +525,7 @@ func (impl AppStoreDeploymentCommonServiceImpl) AddConfigFileToChart(config *uti } // CreateGitOpsRepoAndPushChart is a wrapper for creating gitops repo and pushing chart to created repo -func (impl AppStoreDeploymentCommonServiceImpl) CreateGitOpsRepoAndPushChart(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, builtChartPath string, requirementsConfig *util.ChartConfig, valuesConfig *util.ChartConfig) (*util.ChartGitAttribute, bool, string, error) { +func (impl AppStoreDeploymentCommonServiceImpl) CreateGitOpsRepoAndPushChart(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, builtChartPath string, requirementsConfig *util.ChartConfig, valuesConfig *util.ChartConfig) (*commonBean.ChartGitAttribute, bool, string, error) { repoURL, isNew, err := impl.CreateGitOpsRepo(installAppVersionRequest) if err != nil { impl.logger.Errorw("Error in creating gitops repo for ", "appName", installAppVersionRequest.AppName, "err", err) @@ -542,15 +542,12 @@ func (impl AppStoreDeploymentCommonServiceImpl) CreateGitOpsRepoAndPushChart(ins // CommitConfigToGit is used for committing values.yaml and requirements.yaml file config func (impl AppStoreDeploymentCommonServiceImpl) CommitConfigToGit(chartConfig *util.ChartConfig) (string, error) { - gitOpsConfigBitbucket, err := impl.gitOpsConfigRepository.GetGitOpsConfigByProvider(util.BITBUCKET_PROVIDER) + bitbucketMetadata, err := impl.gitOpsConfigReadService.GetBitbucketMetadata() if err != nil { - if err == pg.ErrNoRows { - gitOpsConfigBitbucket.BitBucketWorkspaceId = "" - } else { - return "", err - } + impl.logger.Errorw("error in getting bitbucket metadata", "err", err) + return "", err } - gitOpsConfig := &bean.GitOpsConfigDto{BitBucketWorkspaceId: gitOpsConfigBitbucket.BitBucketWorkspaceId} + gitOpsConfig := &bean.GitOpsConfigDto{BitBucketWorkspaceId: bitbucketMetadata.BitBucketWorkspaceId} gitHash, _, err := impl.gitFactory.Client.CommitValues(chartConfig, gitOpsConfig) if err != nil { impl.logger.Errorw("error in git commit", "err", err) @@ -578,7 +575,7 @@ func (impl AppStoreDeploymentCommonServiceImpl) GitOpsOperations(manifestRespons impl.logger.Errorw("error in committing dependency config to git", "err", err) return appStoreGitOpsResponse, err } - err = impl.chartTemplateService.GitPull(clonedDir, chartGitAttribute.RepoUrl, appStoreName) + err = impl.gitOpsRemoteOperationService.GitPull(clonedDir, chartGitAttribute.RepoUrl, appStoreName) if err != nil { impl.logger.Errorw("error in git pull", "err", err) return appStoreGitOpsResponse, err @@ -589,7 +586,7 @@ func (impl AppStoreDeploymentCommonServiceImpl) GitOpsOperations(manifestRespons impl.logger.Errorw("error in committing values config to git", "err", err) return appStoreGitOpsResponse, err } - err = impl.chartTemplateService.GitPull(clonedDir, chartGitAttribute.RepoUrl, appStoreName) + err = impl.gitOpsRemoteOperationService.GitPull(clonedDir, chartGitAttribute.RepoUrl, appStoreName) if err != nil { impl.logger.Errorw("error in git pull", "err", err) return appStoreGitOpsResponse, err diff --git a/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go b/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go index 77d651bf19..6293aa0cb6 100644 --- a/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go +++ b/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go @@ -24,13 +24,11 @@ import ( "fmt" "github.com/devtron-labs/common-lib/pubsub-lib" "github.com/devtron-labs/common-lib/pubsub-lib/model" - "path" - "regexp" + commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" "time" - "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/client/argocdServer" - repository3 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/pkg/app/status" appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" @@ -40,18 +38,12 @@ import ( repository5 "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/sql" util2 "github.com/devtron-labs/devtron/pkg/util" - util3 "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/argo" "github.com/go-pg/pg" "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" application2 "github.com/devtron-labs/devtron/client/argocdServer/application" - "github.com/devtron-labs/devtron/client/argocdServer/repository" - "github.com/devtron-labs/devtron/internal/util" "go.uber.org/zap" - "k8s.io/helm/pkg/chartutil" - "k8s.io/helm/pkg/proto/hapi/chart" - "sigs.k8s.io/yaml" ) const ( @@ -60,8 +52,7 @@ const ( ) type AppStoreDeploymentFullModeService interface { - AppStoreDeployOperationGIT(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, *util.ChartGitAttribute, error) - AppStoreDeployOperationACD(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, chartGitAttr *util.ChartGitAttribute, ctx context.Context, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) + AppStoreDeployOperationACD(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, chartGitAttr *commonBean.ChartGitAttribute, ctx context.Context, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) SyncACD(acdAppName string, ctx context.Context) UpdateValuesYaml(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) UpdateRequirementYaml(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, appStoreAppVersion *appStoreDiscoverRepository.AppStoreApplicationVersion) error @@ -70,67 +61,43 @@ type AppStoreDeploymentFullModeService interface { } type AppStoreDeploymentFullModeServiceImpl struct { - logger *zap.SugaredLogger - chartTemplateService util.ChartTemplateService - repositoryService repository.ServiceClient - appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository - environmentRepository repository5.EnvironmentRepository - acdClient application2.ServiceClient - ArgoK8sClient argocdServer.ArgoK8sClient - gitFactory *util.GitFactory - aCDAuthConfig *util2.ACDAuthConfig - globalEnvVariables *util3.GlobalEnvVariables - installedAppRepository repository4.InstalledAppRepository - tokenCache *util2.TokenCache - argoUserService argo.ArgoUserService - gitOpsConfigRepository repository3.GitOpsConfigRepository - pipelineStatusTimelineService status.PipelineStatusTimelineService - appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService - argoClientWrapperService argocdServer.ArgoClientWrapperService - pubSubClient *pubsub_lib.PubSubClientServiceImpl - installedAppRepositoryHistory repository4.InstalledAppVersionHistoryRepository - ACDConfig *argocdServer.ACDConfig + logger *zap.SugaredLogger + acdClient application2.ServiceClient + ArgoK8sClient argocdServer.ArgoK8sClient + aCDAuthConfig *util2.ACDAuthConfig + argoUserService argo.ArgoUserService + pipelineStatusTimelineService status.PipelineStatusTimelineService + appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService + argoClientWrapperService argocdServer.ArgoClientWrapperService + pubSubClient *pubsub_lib.PubSubClientServiceImpl + installedAppRepositoryHistory repository4.InstalledAppVersionHistoryRepository + ACDConfig *argocdServer.ACDConfig + gitOpsConfigReadService config.GitOpsConfigReadService } func NewAppStoreDeploymentFullModeServiceImpl(logger *zap.SugaredLogger, - chartTemplateService util.ChartTemplateService, - repositoryService repository.ServiceClient, - appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository, - environmentRepository repository5.EnvironmentRepository, acdClient application2.ServiceClient, - argoK8sClient argocdServer.ArgoK8sClient, - gitFactory *util.GitFactory, aCDAuthConfig *util2.ACDAuthConfig, - globalEnvVariables *util3.GlobalEnvVariables, - installedAppRepository repository4.InstalledAppRepository, tokenCache *util2.TokenCache, - argoUserService argo.ArgoUserService, gitOpsConfigRepository repository3.GitOpsConfigRepository, - pipelineStatusTimelineService status.PipelineStatusTimelineService, + argoK8sClient argocdServer.ArgoK8sClient, aCDAuthConfig *util2.ACDAuthConfig, + argoUserService argo.ArgoUserService, pipelineStatusTimelineService status.PipelineStatusTimelineService, appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService, argoClientWrapperService argocdServer.ArgoClientWrapperService, pubSubClient *pubsub_lib.PubSubClientServiceImpl, installedAppRepositoryHistory repository4.InstalledAppVersionHistoryRepository, ACDConfig *argocdServer.ACDConfig, -) *AppStoreDeploymentFullModeServiceImpl { + gitOpsConfigReadService config.GitOpsConfigReadService) *AppStoreDeploymentFullModeServiceImpl { appStoreDeploymentFullModeServiceImpl := &AppStoreDeploymentFullModeServiceImpl{ - logger: logger, - chartTemplateService: chartTemplateService, - repositoryService: repositoryService, - appStoreApplicationVersionRepository: appStoreApplicationVersionRepository, - environmentRepository: environmentRepository, - acdClient: acdClient, - ArgoK8sClient: argoK8sClient, - gitFactory: gitFactory, - aCDAuthConfig: aCDAuthConfig, - globalEnvVariables: globalEnvVariables, - installedAppRepository: installedAppRepository, - tokenCache: tokenCache, - argoUserService: argoUserService, - gitOpsConfigRepository: gitOpsConfigRepository, - pipelineStatusTimelineService: pipelineStatusTimelineService, - appStoreDeploymentCommonService: appStoreDeploymentCommonService, - argoClientWrapperService: argoClientWrapperService, - pubSubClient: pubSubClient, - installedAppRepositoryHistory: installedAppRepositoryHistory, - ACDConfig: ACDConfig, + logger: logger, + acdClient: acdClient, + ArgoK8sClient: argoK8sClient, + aCDAuthConfig: aCDAuthConfig, + argoUserService: argoUserService, + pipelineStatusTimelineService: pipelineStatusTimelineService, + appStoreDeploymentCommonService: appStoreDeploymentCommonService, + argoClientWrapperService: argoClientWrapperService, + pubSubClient: pubSubClient, + installedAppRepositoryHistory: installedAppRepositoryHistory, + ACDConfig: ACDConfig, + gitOpsConfigReadService: gitOpsConfigReadService, } err := appStoreDeploymentFullModeServiceImpl.SubscribeHelmInstallStatus() if err != nil { @@ -139,182 +106,7 @@ func NewAppStoreDeploymentFullModeServiceImpl(logger *zap.SugaredLogger, return appStoreDeploymentFullModeServiceImpl } -func (impl AppStoreDeploymentFullModeServiceImpl) AppStoreDeployOperationGIT(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, *util.ChartGitAttribute, error) { - appStoreAppVersion, err := impl.appStoreApplicationVersionRepository.FindById(installAppVersionRequest.AppStoreVersion) - if err != nil { - impl.logger.Errorw("fetching error", "err", err) - return installAppVersionRequest, nil, err - } - - environment, err := impl.environmentRepository.FindById(installAppVersionRequest.EnvironmentId) - if err != nil { - impl.logger.Errorw("fetching error", "err", err) - return installAppVersionRequest, nil, err - } - - //STEP 1: Commit and PUSH on Gitlab - template := appStoreBean.CHART_PROXY_TEMPLATE - chartPath := path.Join(appStoreBean.RefChartProxyDirPath, template) - valid, err := chartutil.IsChartDir(chartPath) - if err != nil || !valid { - impl.logger.Errorw("invalid base chart", "dir", chartPath, "err", err) - return installAppVersionRequest, nil, err - } - chartMeta := &chart.Metadata{ - Name: installAppVersionRequest.AppName, - Version: "1.0.1", - } - _, chartGitAttr, err := impl.chartTemplateService.CreateChartProxy(chartMeta, chartPath, environment.Name, installAppVersionRequest) - if err != nil { - return installAppVersionRequest, nil, err - } - - //STEP 3 - update requirements and values - argocdAppName := installAppVersionRequest.AppName + "-" + environment.Name - dependency := appStoreBean.Dependency{ - Name: appStoreAppVersion.AppStore.Name, - Version: appStoreAppVersion.Version, - } - if appStoreAppVersion.AppStore.ChartRepo != nil { - dependency.Repository = appStoreAppVersion.AppStore.ChartRepo.Url - } - var dependencies []appStoreBean.Dependency - dependencies = append(dependencies, dependency) - requirementDependencies := &appStoreBean.Dependencies{ - Dependencies: dependencies, - } - requirementDependenciesByte, err := json.Marshal(requirementDependencies) - if err != nil { - return installAppVersionRequest, nil, err - } - requirementDependenciesByte, err = yaml.JSONToYAML(requirementDependenciesByte) - if err != nil { - return installAppVersionRequest, nil, err - } - - gitOpsRepoName := impl.chartTemplateService.GetGitOpsRepoName(installAppVersionRequest.AppName) - //getting username & emailId for commit author data - userEmailId, userName := impl.chartTemplateService.GetUserEmailIdAndNameForGitOpsCommit(installAppVersionRequest.UserId) - gitOpsConfigBitbucket, err := impl.gitOpsConfigRepository.GetGitOpsConfigByProvider(util.BITBUCKET_PROVIDER) - if err != nil { - if err == pg.ErrNoRows { - gitOpsConfigBitbucket.BitBucketWorkspaceId = "" - } else { - return installAppVersionRequest, nil, err - } - } - requirmentYamlConfig := &util.ChartConfig{ - FileName: appStoreBean.REQUIREMENTS_YAML_FILE, - FileContent: string(requirementDependenciesByte), - ChartName: chartMeta.Name, - ChartLocation: argocdAppName, - ChartRepoName: gitOpsRepoName, - ReleaseMessage: fmt.Sprintf("release-%d-env-%d ", appStoreAppVersion.Id, environment.Id), - UserEmailId: userEmailId, - UserName: userName, - } - gitOpsConfig := &bean.GitOpsConfigDto{BitBucketWorkspaceId: gitOpsConfigBitbucket.BitBucketWorkspaceId} - _, _, err = impl.gitFactory.Client.CommitValues(requirmentYamlConfig, gitOpsConfig) - if err != nil { - impl.logger.Errorw("error in git commit", "err", err) - return installAppVersionRequest, nil, err - } - - //GIT PULL - space := regexp.MustCompile(`\s+`) - appStoreName := space.ReplaceAllString(chartMeta.Name, "-") - clonedDir := impl.gitFactory.GitWorkingDir + "" + appStoreName - err = impl.chartTemplateService.GitPull(clonedDir, chartGitAttr.RepoUrl, appStoreName) - if err != nil { - impl.logger.Errorw("error in git pull", "err", err) - return installAppVersionRequest, nil, err - } - - //update values yaml in chart - ValuesOverrideByte, err := yaml.YAMLToJSON([]byte(installAppVersionRequest.ValuesOverrideYaml)) - if err != nil { - impl.logger.Errorw("error in json patch", "err", err) - return installAppVersionRequest, nil, err - } - - var dat map[string]interface{} - err = json.Unmarshal(ValuesOverrideByte, &dat) - - valuesMap := make(map[string]map[string]interface{}) - valuesMap[appStoreAppVersion.AppStore.Name] = dat - valuesByte, err := json.Marshal(valuesMap) - if err != nil { - impl.logger.Errorw("error in marshaling", "err", err) - return installAppVersionRequest, nil, err - } - - valuesYamlConfig := &util.ChartConfig{ - FileName: appStoreBean.VALUES_YAML_FILE, - FileContent: string(valuesByte), - ChartName: chartMeta.Name, - ChartLocation: argocdAppName, - ChartRepoName: gitOpsRepoName, - ReleaseMessage: fmt.Sprintf("release-%d-env-%d ", appStoreAppVersion.Id, environment.Id), - UserEmailId: userEmailId, - UserName: userName, - } - - commitHash, _, err := impl.gitFactory.Client.CommitValues(valuesYamlConfig, gitOpsConfig) - if err != nil { - impl.logger.Errorw("error in git commit", "err", err) - //update timeline status for git commit failed state - gitCommitStatus := pipelineConfig.TIMELINE_STATUS_GIT_COMMIT_FAILED - gitCommitStatusDetail := fmt.Sprintf("Git commit failed - %v", err) - timeline := &pipelineConfig.PipelineStatusTimeline{ - InstalledAppVersionHistoryId: installAppVersionRequest.InstalledAppVersionHistoryId, - Status: gitCommitStatus, - StatusDetail: gitCommitStatusDetail, - StatusTime: time.Now(), - AuditLog: sql.AuditLog{ - CreatedBy: installAppVersionRequest.UserId, - CreatedOn: time.Now(), - UpdatedBy: installAppVersionRequest.UserId, - UpdatedOn: time.Now(), - }, - } - timelineErr := impl.pipelineStatusTimelineService.SaveTimeline(timeline, tx, true) - if timelineErr != nil { - impl.logger.Errorw("error in creating timeline status for git commit", "err", timelineErr, "timeline", timeline) - } - return installAppVersionRequest, nil, err - } - //creating timeline for Git Commit stage - timeline := &pipelineConfig.PipelineStatusTimeline{ - InstalledAppVersionHistoryId: installAppVersionRequest.InstalledAppVersionHistoryId, - Status: pipelineConfig.TIMELINE_STATUS_GIT_COMMIT, - StatusDetail: "Git commit done successfully.", - StatusTime: time.Now(), - AuditLog: sql.AuditLog{ - CreatedBy: installAppVersionRequest.UserId, - CreatedOn: time.Now(), - UpdatedBy: installAppVersionRequest.UserId, - UpdatedOn: time.Now(), - }, - } - err = impl.pipelineStatusTimelineService.SaveTimeline(timeline, tx, true) - if err != nil { - impl.logger.Errorw("error in creating timeline status for git commit", "err", err, "timeline", timeline) - } - - //sync local dir with remote - err = impl.chartTemplateService.GitPull(clonedDir, chartGitAttr.RepoUrl, appStoreName) - if err != nil { - impl.logger.Errorw("error in git pull", "err", err) - return installAppVersionRequest, nil, err - } - installAppVersionRequest.GitHash = commitHash - installAppVersionRequest.ACDAppName = argocdAppName - installAppVersionRequest.Environment = environment - - return installAppVersionRequest, chartGitAttr, nil -} - -func (impl AppStoreDeploymentFullModeServiceImpl) AppStoreDeployOperationACD(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, chartGitAttr *util.ChartGitAttribute, ctx context.Context, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) { +func (impl AppStoreDeploymentFullModeServiceImpl) AppStoreDeployOperationACD(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, chartGitAttr *commonBean.ChartGitAttribute, ctx context.Context, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) { ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) defer cancel() //STEP 4: registerInArgo @@ -373,7 +165,7 @@ func (impl AppStoreDeploymentFullModeServiceImpl) SyncACD(acdAppName string, ctx } } -func (impl AppStoreDeploymentFullModeServiceImpl) createInArgo(chartGitAttribute *util.ChartGitAttribute, ctx context.Context, envModel repository5.Environment, argocdAppName string) error { +func (impl AppStoreDeploymentFullModeServiceImpl) createInArgo(chartGitAttribute *commonBean.ChartGitAttribute, ctx context.Context, envModel repository5.Environment, argocdAppName string) error { appNamespace := envModel.Namespace if appNamespace == "" { appNamespace = "default" @@ -416,7 +208,7 @@ func (impl AppStoreDeploymentFullModeServiceImpl) GetGitOpsRepoName(appName stri } if application != nil { gitOpsRepoUrl := application.Spec.Source.RepoURL - gitOpsRepoName = impl.chartTemplateService.GetGitOpsRepoNameFromUrl(gitOpsRepoUrl) + gitOpsRepoName = impl.gitOpsConfigReadService.GetGitOpsRepoNameFromUrl(gitOpsRepoUrl) } return gitOpsRepoName, nil } diff --git a/pkg/appStore/deployment/service/AppStoreDeploymentService.go b/pkg/appStore/deployment/service/AppStoreDeploymentService.go index 02e55ca006..89c61c6d09 100644 --- a/pkg/appStore/deployment/service/AppStoreDeploymentService.go +++ b/pkg/appStore/deployment/service/AppStoreDeploymentService.go @@ -39,11 +39,11 @@ import ( appStoreDeploymentTool "github.com/devtron-labs/devtron/pkg/appStore/deployment/tool" appStoreDeploymentGitopsTool "github.com/devtron-labs/devtron/pkg/appStore/deployment/tool/gitops" appStoreDiscoverRepository "github.com/devtron-labs/devtron/pkg/appStore/discover/repository" - "github.com/devtron-labs/devtron/pkg/attributes" "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/cluster" cluster2 "github.com/devtron-labs/devtron/pkg/cluster" clusterRepository "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" "github.com/devtron-labs/devtron/pkg/sql" util2 "github.com/devtron-labs/devtron/util" "github.com/go-pg/pg" @@ -106,8 +106,8 @@ type AppStoreDeploymentServiceImpl struct { installedAppRepositoryHistory repository.InstalledAppVersionHistoryRepository gitOpsRepository repository2.GitOpsConfigRepository deploymentTypeConfig *DeploymentServiceTypeConfig - ChartTemplateService util.ChartTemplateService aCDConfig *argocdServer.ACDConfig + gitOpsConfigReadService config.GitOpsConfigReadService } func NewAppStoreDeploymentServiceImpl(logger *zap.SugaredLogger, installedAppRepository repository.InstalledAppRepository, @@ -117,8 +117,9 @@ func NewAppStoreDeploymentServiceImpl(logger *zap.SugaredLogger, installedAppRep appStoreDeploymentArgoCdService appStoreDeploymentGitopsTool.AppStoreDeploymentArgoCdService, environmentService cluster.EnvironmentService, clusterService cluster.ClusterService, helmAppService client.HelmAppService, appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService, globalEnvVariables *util2.GlobalEnvVariables, - installedAppRepositoryHistory repository.InstalledAppVersionHistoryRepository, gitOpsRepository repository2.GitOpsConfigRepository, attributesService attributes.AttributesService, - deploymentTypeConfig *DeploymentServiceTypeConfig, ChartTemplateService util.ChartTemplateService, aCDConfig *argocdServer.ACDConfig) *AppStoreDeploymentServiceImpl { + installedAppRepositoryHistory repository.InstalledAppVersionHistoryRepository, gitOpsRepository repository2.GitOpsConfigRepository, + deploymentTypeConfig *DeploymentServiceTypeConfig, aCDConfig *argocdServer.ACDConfig, + gitOpsConfigReadService config.GitOpsConfigReadService) *AppStoreDeploymentServiceImpl { appStoreDeploymentServiceImpl := &AppStoreDeploymentServiceImpl{ logger: logger, @@ -138,8 +139,8 @@ func NewAppStoreDeploymentServiceImpl(logger *zap.SugaredLogger, installedAppRep installedAppRepositoryHistory: installedAppRepositoryHistory, gitOpsRepository: gitOpsRepository, deploymentTypeConfig: deploymentTypeConfig, - ChartTemplateService: ChartTemplateService, aCDConfig: aCDConfig, + gitOpsConfigReadService: gitOpsConfigReadService, } return appStoreDeploymentServiceImpl } @@ -247,7 +248,7 @@ func (impl AppStoreDeploymentServiceImpl) AppStoreDeployOperationDB(installAppVe installedAppModel.UpdatedOn = time.Now() installedAppModel.Active = true if util2.IsFullStack() { - installedAppModel.GitOpsRepoName = impl.ChartTemplateService.GetGitOpsRepoName(installAppVersionRequest.AppName) + installedAppModel.GitOpsRepoName = impl.gitOpsConfigReadService.GetGitOpsRepoName(installAppVersionRequest.AppName) installAppVersionRequest.GitOpsRepoName = installedAppModel.GitOpsRepoName } installedApp, err := impl.installedAppRepository.CreateInstalledApp(installedAppModel, tx) @@ -1289,11 +1290,11 @@ func (impl *AppStoreDeploymentServiceImpl) CheckIfMonoRepoMigrationRequired(inst return false } } else { - gitOpsRepoName = impl.ChartTemplateService.GetGitOpsRepoName(installAppVersionRequest.AppName) + gitOpsRepoName = impl.gitOpsConfigReadService.GetGitOpsRepoName(installAppVersionRequest.AppName) } } //here will set new git repo name if required to migrate - newGitOpsRepoName := impl.ChartTemplateService.GetGitOpsRepoName(installedApp.App.AppName) + newGitOpsRepoName := impl.gitOpsConfigReadService.GetGitOpsRepoName(installedApp.App.AppName) //checking weather git repo migration needed or not, if existing git repo and new independent git repo is not same than go ahead with migration if newGitOpsRepoName != gitOpsRepoName { monoRepoMigrationRequired = true @@ -1447,11 +1448,11 @@ func (impl *AppStoreDeploymentServiceImpl) UpdateInstalledApp(ctx context.Contex return nil, err } } else { - gitOpsRepoName = impl.ChartTemplateService.GetGitOpsRepoName(installAppVersionRequest.AppName) + gitOpsRepoName = impl.gitOpsConfigReadService.GetGitOpsRepoName(installAppVersionRequest.AppName) } } //here will set new git repo name if required to migrate - newGitOpsRepoName := impl.ChartTemplateService.GetGitOpsRepoName(installedApp.App.AppName) + newGitOpsRepoName := impl.gitOpsConfigReadService.GetGitOpsRepoName(installedApp.App.AppName) //checking weather git repo migration needed or not, if existing git repo and new independent git repo is not same than go ahead with migration if newGitOpsRepoName != gitOpsRepoName { monoRepoMigrationRequired = true diff --git a/pkg/appStore/deployment/service/InstalledAppService.go b/pkg/appStore/deployment/service/InstalledAppService.go index f97b19e5eb..eb1c89ad53 100644 --- a/pkg/appStore/deployment/service/InstalledAppService.go +++ b/pkg/appStore/deployment/service/InstalledAppService.go @@ -20,6 +20,9 @@ package service import ( "bytes" "context" + commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" + /* #nosec */ "crypto/sha1" "encoding/json" @@ -107,7 +110,6 @@ type InstalledAppService interface { type InstalledAppServiceImpl struct { logger *zap.SugaredLogger installedAppRepository repository2.InstalledAppRepository - chartTemplateService util.ChartTemplateService repositoryService repository.ServiceClient appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository environmentRepository repository5.EnvironmentRepository @@ -137,11 +139,11 @@ type InstalledAppServiceImpl struct { k8sCommonService k8s.K8sCommonService k8sApplicationService application3.K8sApplicationService acdConfig *argocdServer.ACDConfig + gitOpsConfigReadService config.GitOpsConfigReadService } func NewInstalledAppServiceImpl(logger *zap.SugaredLogger, installedAppRepository repository2.InstalledAppRepository, - chartTemplateService util.ChartTemplateService, repositoryService repository.ServiceClient, appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository, environmentRepository repository5.EnvironmentRepository, teamRepository repository4.TeamRepository, @@ -161,11 +163,11 @@ func NewInstalledAppServiceImpl(logger *zap.SugaredLogger, pipelineStatusTimelineService status.PipelineStatusTimelineService, appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService, k8sCommonService k8s.K8sCommonService, k8sApplicationService application3.K8sApplicationService, - acdConfig *argocdServer.ACDConfig) (*InstalledAppServiceImpl, error) { + acdConfig *argocdServer.ACDConfig, + gitOpsConfigReadService config.GitOpsConfigReadService) (*InstalledAppServiceImpl, error) { impl := &InstalledAppServiceImpl{ logger: logger, installedAppRepository: installedAppRepository, - chartTemplateService: chartTemplateService, repositoryService: repositoryService, appStoreApplicationVersionRepository: appStoreApplicationVersionRepository, environmentRepository: environmentRepository, @@ -195,6 +197,7 @@ func NewInstalledAppServiceImpl(logger *zap.SugaredLogger, k8sCommonService: k8sCommonService, k8sApplicationService: k8sApplicationService, acdConfig: acdConfig, + gitOpsConfigReadService: gitOpsConfigReadService, } err := impl.Subscribe() if err != nil { @@ -383,7 +386,7 @@ func (impl InstalledAppServiceImpl) createChartGroupEntryObject(installAppVersio } func (impl InstalledAppServiceImpl) performDeployStageOnAcd(installedAppVersion *appStoreBean.InstallAppVersionDTO, ctx context.Context, userId int32) (*appStoreBean.InstallAppVersionDTO, error) { installedAppVersion.ACDAppName = fmt.Sprintf("%s-%s", installedAppVersion.AppName, installedAppVersion.Environment.Name) - chartGitAttr := &util.ChartGitAttribute{} + chartGitAttr := &commonBean.ChartGitAttribute{} if installedAppVersion.Status == appStoreBean.DEPLOY_INIT || installedAppVersion.Status == appStoreBean.ENQUEUED || installedAppVersion.Status == appStoreBean.QUE_ERROR || @@ -472,19 +475,15 @@ func (impl InstalledAppServiceImpl) performDeployStageOnAcd(installedAppVersion impl.logger.Errorw("fetching error", "err", err) return nil, err } - gitOpsConfigBitbucket, err := impl.gitOpsRepository.GetGitOpsConfigByProvider(util.BITBUCKET_PROVIDER) + bitbucketMetadata, err := impl.gitOpsConfigReadService.GetBitbucketMetadata() if err != nil { - if err == pg.ErrNoRows { - gitOpsConfigBitbucket.BitBucketWorkspaceId = "" - gitOpsConfigBitbucket.BitBucketProjectKey = "" - } else { - return nil, err - } + impl.logger.Errorw("error in getting bitbucket metadata", "err", err) + return nil, err } config := &bean2.GitOpsConfigDto{ GitRepoName: installedAppVersion.GitOpsRepoName, - BitBucketWorkspaceId: gitOpsConfigBitbucket.BitBucketProjectKey, - BitBucketProjectKey: gitOpsConfigBitbucket.BitBucketProjectKey, + BitBucketWorkspaceId: bitbucketMetadata.BitBucketWorkspaceId, + BitBucketProjectKey: bitbucketMetadata.BitBucketProjectKey, } repoUrl, err := impl.gitFactory.Client.GetRepoUrl(config) if err != nil { diff --git a/pkg/appStore/deployment/tool/AppStoreDeploymentHelmService.go b/pkg/appStore/deployment/tool/AppStoreDeploymentHelmService.go index 9716abb607..c317353f48 100644 --- a/pkg/appStore/deployment/tool/AppStoreDeploymentHelmService.go +++ b/pkg/appStore/deployment/tool/AppStoreDeploymentHelmService.go @@ -4,6 +4,7 @@ import ( "context" "errors" repository2 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" + commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" "net/http" "time" @@ -23,7 +24,7 @@ import ( ) type AppStoreDeploymentHelmService interface { - InstallApp(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, chartGitAttr *util.ChartGitAttribute, ctx context.Context, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) + InstallApp(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, chartGitAttr *commonBean.ChartGitAttribute, ctx context.Context, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) GetAppStatus(installedAppAndEnvDetails repository.InstalledAppAndEnvDetails, w http.ResponseWriter, r *http.Request, token string) (string, error) DeleteInstalledApp(ctx context.Context, appName string, environmentName string, installAppVersionRequest *appStoreBean.InstallAppVersionDTO, installedApps *repository.InstalledApps, dbTransaction *pg.Tx) error RollbackRelease(ctx context.Context, installedApp *appStoreBean.InstallAppVersionDTO, deploymentVersion int32, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, bool, error) @@ -36,7 +37,7 @@ type AppStoreDeploymentHelmService interface { DeleteDeploymentApp(ctx context.Context, appName string, environmentName string, installAppVersionRequest *appStoreBean.InstallAppVersionDTO) error UpdateInstalledAppAndPipelineStatusForFailedDeploymentStatus(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, triggeredAt time.Time, err error) error SaveTimelineForACDHelmApps(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, status string, statusDetail string, statusTime time.Time, tx *pg.Tx) error - UpdateChartInfo(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, ChartGitAttribute *util.ChartGitAttribute, installedAppVersionHistoryId int, ctx context.Context) error + UpdateChartInfo(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, ChartGitAttribute *commonBean.ChartGitAttribute, installedAppVersionHistoryId int, ctx context.Context) error } type AppStoreDeploymentHelmServiceImpl struct { @@ -64,7 +65,7 @@ func NewAppStoreDeploymentHelmServiceImpl(logger *zap.SugaredLogger, helmAppServ } } -func (impl AppStoreDeploymentHelmServiceImpl) UpdateChartInfo(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, ChartGitAttribute *util.ChartGitAttribute, installedAppVersionHistoryId int, ctx context.Context) error { +func (impl AppStoreDeploymentHelmServiceImpl) UpdateChartInfo(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, ChartGitAttribute *commonBean.ChartGitAttribute, installedAppVersionHistoryId int, ctx context.Context) error { err := impl.updateApplicationWithChartInfo(ctx, installAppVersionRequest.InstalledAppId, installAppVersionRequest.AppStoreVersion, installAppVersionRequest.ValuesOverrideYaml, installedAppVersionHistoryId) if err != nil { impl.Logger.Errorw("error in updating helm app", "err", err) @@ -73,7 +74,7 @@ func (impl AppStoreDeploymentHelmServiceImpl) UpdateChartInfo(installAppVersionR return nil } -func (impl AppStoreDeploymentHelmServiceImpl) InstallApp(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, chartGitAttr *util.ChartGitAttribute, ctx context.Context, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) { +func (impl AppStoreDeploymentHelmServiceImpl) InstallApp(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, chartGitAttr *commonBean.ChartGitAttribute, ctx context.Context, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) { installAppVersionRequest.DeploymentAppType = util.PIPELINE_DEPLOYMENT_TYPE_HELM appStoreAppVersion, err := impl.appStoreApplicationVersionRepository.FindById(installAppVersionRequest.AppStoreVersion) if err != nil { @@ -476,6 +477,6 @@ func (impl *AppStoreDeploymentHelmServiceImpl) UpdateInstalledAppAndPipelineStat // TODO: Need to refactor this,refer below reason // This is being done as in ea mode wire argocd service is being binded to helmServiceImpl due to which we are restricted to implement this here. // RefreshAndUpdateACDApp this will update chart info in acd app if required in case of mono repo migration and will refresh argo app -func (impl *AppStoreDeploymentHelmServiceImpl) UpdateAndSyncACDApps(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, ChartGitAttribute *util.ChartGitAttribute, isMonoRepoMigrationRequired bool, ctx context.Context, tx *pg.Tx) error { +func (impl *AppStoreDeploymentHelmServiceImpl) UpdateAndSyncACDApps(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, ChartGitAttribute *commonBean.ChartGitAttribute, isMonoRepoMigrationRequired bool, ctx context.Context, tx *pg.Tx) error { return errors.New("this is not implemented") } diff --git a/pkg/appStore/deployment/tool/gitops/AppStoreDeploymentArgoCdService.go b/pkg/appStore/deployment/tool/gitops/AppStoreDeploymentArgoCdService.go index 3b39254642..9bfcf39a40 100644 --- a/pkg/appStore/deployment/tool/gitops/AppStoreDeploymentArgoCdService.go +++ b/pkg/appStore/deployment/tool/gitops/AppStoreDeploymentArgoCdService.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" "net/http" "strings" "time" @@ -18,7 +19,6 @@ import ( "github.com/devtron-labs/devtron/client/argocdServer" application2 "github.com/devtron-labs/devtron/client/argocdServer/application" "github.com/devtron-labs/devtron/internal/constants" - repository3 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app/status" @@ -43,7 +43,7 @@ import ( type AppStoreDeploymentArgoCdService interface { //InstallApp(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, ctx context.Context) (*appStoreBean.InstallAppVersionDTO, error) - InstallApp(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, chartGitAttr *util.ChartGitAttribute, ctx context.Context, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) + InstallApp(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, chartGitAttr *commonBean.ChartGitAttribute, ctx context.Context, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) GetAppStatus(installedAppAndEnvDetails repository.InstalledAppAndEnvDetails, w http.ResponseWriter, r *http.Request, token string) (string, error) DeleteInstalledApp(ctx context.Context, appName string, environmentName string, installAppVersionRequest *appStoreBean.InstallAppVersionDTO, installedApps *repository.InstalledApps, dbTransaction *pg.Tx) error RollbackRelease(ctx context.Context, installedApp *appStoreBean.InstallAppVersionDTO, deploymentVersion int32, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, bool, error) @@ -56,8 +56,8 @@ type AppStoreDeploymentArgoCdService interface { DeleteDeploymentApp(ctx context.Context, appName string, environmentName string, installAppVersionRequest *appStoreBean.InstallAppVersionDTO) error UpdateInstalledAppAndPipelineStatusForFailedDeploymentStatus(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, triggeredAt time.Time, err error) error SaveTimelineForACDHelmApps(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, status string, statusDetail string, statusTime time.Time, tx *pg.Tx) error - UpdateChartInfo(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, ChartGitAttribute *util.ChartGitAttribute, installedAppVersionHistoryId int, ctx context.Context) error - UpdateAndSyncACDApps(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, ChartGitAttribute *util.ChartGitAttribute, isMonoRepoMigrationRequired bool, ctx context.Context, tx *pg.Tx) error + UpdateChartInfo(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, ChartGitAttribute *commonBean.ChartGitAttribute, installedAppVersionHistoryId int, ctx context.Context) error + UpdateAndSyncACDApps(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, ChartGitAttribute *commonBean.ChartGitAttribute, isMonoRepoMigrationRequired bool, ctx context.Context, tx *pg.Tx) error } type AppStoreDeploymentArgoCdServiceImpl struct { @@ -67,12 +67,10 @@ type AppStoreDeploymentArgoCdServiceImpl struct { chartGroupDeploymentRepository repository.ChartGroupDeploymentRepository installedAppRepository repository.InstalledAppRepository installedAppRepositoryHistory repository.InstalledAppVersionHistoryRepository - chartTemplateService util.ChartTemplateService gitFactory *util.GitFactory argoUserService argo.ArgoUserService appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService helmAppService client.HelmAppService - gitOpsConfigRepository repository3.GitOpsConfigRepository appStatusService appStatus.AppStatusService pipelineStatusTimelineService status.PipelineStatusTimelineService userService user.UserService @@ -84,9 +82,9 @@ type AppStoreDeploymentArgoCdServiceImpl struct { func NewAppStoreDeploymentArgoCdServiceImpl(logger *zap.SugaredLogger, appStoreDeploymentFullModeService appStoreDeploymentFullMode.AppStoreDeploymentFullModeService, acdClient application2.ServiceClient, chartGroupDeploymentRepository repository.ChartGroupDeploymentRepository, - installedAppRepository repository.InstalledAppRepository, installedAppRepositoryHistory repository.InstalledAppVersionHistoryRepository, chartTemplateService util.ChartTemplateService, + installedAppRepository repository.InstalledAppRepository, installedAppRepositoryHistory repository.InstalledAppVersionHistoryRepository, gitFactory *util.GitFactory, argoUserService argo.ArgoUserService, appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService, - helmAppService client.HelmAppService, gitOpsConfigRepository repository3.GitOpsConfigRepository, appStatusService appStatus.AppStatusService, + helmAppService client.HelmAppService, appStatusService appStatus.AppStatusService, pipelineStatusTimelineService status.PipelineStatusTimelineService, userService user.UserService, pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository, appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository, @@ -99,12 +97,10 @@ func NewAppStoreDeploymentArgoCdServiceImpl(logger *zap.SugaredLogger, appStoreD chartGroupDeploymentRepository: chartGroupDeploymentRepository, installedAppRepository: installedAppRepository, installedAppRepositoryHistory: installedAppRepositoryHistory, - chartTemplateService: chartTemplateService, gitFactory: gitFactory, argoUserService: argoUserService, appStoreDeploymentCommonService: appStoreDeploymentCommonService, helmAppService: helmAppService, - gitOpsConfigRepository: gitOpsConfigRepository, appStatusService: appStatusService, pipelineStatusTimelineService: pipelineStatusTimelineService, userService: userService, @@ -116,7 +112,7 @@ func NewAppStoreDeploymentArgoCdServiceImpl(logger *zap.SugaredLogger, appStoreD } // UpdateAndSyncACDApps this will update chart info in acd app if required in case of mono repo migration and will refresh argo app -func (impl AppStoreDeploymentArgoCdServiceImpl) UpdateAndSyncACDApps(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, ChartGitAttribute *util.ChartGitAttribute, isMonoRepoMigrationRequired bool, ctx context.Context, tx *pg.Tx) error { +func (impl AppStoreDeploymentArgoCdServiceImpl) UpdateAndSyncACDApps(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, ChartGitAttribute *commonBean.ChartGitAttribute, isMonoRepoMigrationRequired bool, ctx context.Context, tx *pg.Tx) error { if isMonoRepoMigrationRequired { // update repo details on ArgoCD as repo is changed err := impl.UpdateChartInfo(installAppVersionRequest, ChartGitAttribute, 0, ctx) @@ -154,7 +150,7 @@ func (impl AppStoreDeploymentArgoCdServiceImpl) UpdateAndSyncACDApps(installAppV } // UpdateChartInfo this will update chart info in acd app, needed when repo for an app is changed -func (impl AppStoreDeploymentArgoCdServiceImpl) UpdateChartInfo(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, ChartGitAttribute *util.ChartGitAttribute, installedAppVersionHistoryId int, ctx context.Context) error { +func (impl AppStoreDeploymentArgoCdServiceImpl) UpdateChartInfo(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, ChartGitAttribute *commonBean.ChartGitAttribute, installedAppVersionHistoryId int, ctx context.Context) error { installAppVersionRequest, err := impl.patchAcdApp(ctx, installAppVersionRequest, ChartGitAttribute) if err != nil { return err @@ -187,7 +183,7 @@ func (impl AppStoreDeploymentArgoCdServiceImpl) SaveTimelineForACDHelmApps(insta return timelineErr } -func (impl AppStoreDeploymentArgoCdServiceImpl) InstallApp(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, chartGitAttr *util.ChartGitAttribute, ctx context.Context, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) { +func (impl AppStoreDeploymentArgoCdServiceImpl) InstallApp(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, chartGitAttr *commonBean.ChartGitAttribute, ctx context.Context, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) { installAppVersionRequest, err := impl.appStoreDeploymentFullModeService.AppStoreDeployOperationACD(installAppVersionRequest, chartGitAttr, ctx, tx) if err != nil { @@ -635,7 +631,7 @@ func (impl AppStoreDeploymentArgoCdServiceImpl) UpdateInstalledApp(ctx context.C impl.Logger.Errorw("error in creating timeline status for deployment initiation for update of installedAppVersionHistoryId", "err", err, "installedAppVersionHistoryId", installAppVersionRequest.InstalledAppVersionHistoryId) } //update values yaml in chart - installAppVersionRequest, err = impl.updateValuesYaml(environment, installedAppVersion, installAppVersionRequest, tx) + installAppVersionRequest, err = impl.updateValuesYaml(installAppVersionRequest, tx) if err != nil { impl.Logger.Errorw("error while commit values to git", "error", err) noTargetFound, _ := impl.appStoreDeploymentCommonService.ParseGitRepoErrorResponse(err) @@ -658,7 +654,7 @@ func (impl AppStoreDeploymentArgoCdServiceImpl) UpdateInstalledApp(ctx context.C return installAppVersionRequest, nil } -func (impl AppStoreDeploymentArgoCdServiceImpl) patchAcdApp(ctx context.Context, installAppVersionRequest *appStoreBean.InstallAppVersionDTO, chartGitAttr *util.ChartGitAttribute) (*appStoreBean.InstallAppVersionDTO, error) { +func (impl AppStoreDeploymentArgoCdServiceImpl) patchAcdApp(ctx context.Context, installAppVersionRequest *appStoreBean.InstallAppVersionDTO, chartGitAttr *commonBean.ChartGitAttribute) (*appStoreBean.InstallAppVersionDTO, error) { ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) defer cancel() //registerInArgo @@ -684,8 +680,7 @@ func (impl AppStoreDeploymentArgoCdServiceImpl) patchAcdApp(ctx context.Context, return installAppVersionRequest, nil } -func (impl AppStoreDeploymentArgoCdServiceImpl) updateValuesYaml(environment *clusterRepository.Environment, installedAppVersion *repository.InstalledAppVersions, - installAppVersionRequest *appStoreBean.InstallAppVersionDTO, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) { +func (impl AppStoreDeploymentArgoCdServiceImpl) updateValuesYaml(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) { appStoreAppVersion, err := impl.appStoreApplicationVersionRepository.FindById(installAppVersionRequest.AppStoreVersion) if err != nil { diff --git a/pkg/chart/ChartService.go b/pkg/chart/ChartService.go index 9939447811..4a9430fc46 100644 --- a/pkg/chart/ChartService.go +++ b/pkg/chart/ChartService.go @@ -67,6 +67,8 @@ type ChartService interface { PatchEnvOverrides(values json.RawMessage, oldChartType string, newChartType string) (json.RawMessage, error) ChartRefAutocompleteForAppOrEnv(appId int, envId int) (*bean2.ChartRefAutocompleteResponse, error) + + UpdateGitRepoUrlInCharts(appId int, repoUrl, chartLocation string, userId int32) error } type ChartServiceImpl struct { @@ -197,7 +199,7 @@ func (impl ChartServiceImpl) Create(templateRequest TemplateRequest, ctx context if err != nil { return nil, err } - chartValues, _, err := impl.chartTemplateService.FetchValuesFromReferenceChart(chartMeta, refChart, templateName, templateRequest.UserId, pipelineStrategyPath) + chartValues, err := impl.chartTemplateService.FetchValuesFromReferenceChart(chartMeta, refChart, templateName, templateRequest.UserId, pipelineStrategyPath) if err != nil { return nil, err } @@ -320,7 +322,7 @@ func (impl ChartServiceImpl) CreateChartFromEnvOverride(templateRequest Template if err != nil { return nil, err } - chartValues, _, err := impl.chartTemplateService.FetchValuesFromReferenceChart(chartMeta, refChart, templateName, templateRequest.UserId, pipelineStrategyPath) + chartValues, err := impl.chartTemplateService.FetchValuesFromReferenceChart(chartMeta, refChart, templateName, templateRequest.UserId, pipelineStrategyPath) if err != nil { return nil, err } @@ -819,3 +821,23 @@ func (impl ChartServiceImpl) CheckIfChartRefUserUploadedByAppId(id int) (bool, e } return chartData.UserUploaded, err } + +func (impl *ChartServiceImpl) UpdateGitRepoUrlInCharts(appId int, repoUrl, chartLocation string, userId int32) error { + charts, err := impl.chartRepository.FindActiveChartsByAppId(appId) + if err != nil && pg.ErrNoRows != err { + return err + } + for _, ch := range charts { + if len(ch.GitRepoUrl) == 0 { + ch.GitRepoUrl = repoUrl + ch.ChartLocation = chartLocation + ch.UpdatedOn = time.Now() + ch.UpdatedBy = userId + err = impl.chartRepository.Update(ch) + if err != nil { + return err + } + } + } + return nil +} diff --git a/pkg/commonService/CommonService.go b/pkg/commonService/CommonService.go index c60e37d0de..4569dad013 100644 --- a/pkg/commonService/CommonService.go +++ b/pkg/commonService/CommonService.go @@ -40,7 +40,6 @@ type CommonServiceImpl struct { logger *zap.SugaredLogger chartRepository chartRepoRepository.ChartRepository environmentConfigRepository chartConfig.EnvConfigOverrideRepository - gitOpsRepository repository.GitOpsConfigRepository dockerReg dockerRegistryRepository.DockerArtifactStoreRepository attributeRepo repository.AttributesRepository gitProviderRepository repository.GitProviderRepository @@ -52,7 +51,6 @@ type CommonServiceImpl struct { func NewCommonServiceImpl(logger *zap.SugaredLogger, chartRepository chartRepoRepository.ChartRepository, environmentConfigRepository chartConfig.EnvConfigOverrideRepository, - gitOpsRepository repository.GitOpsConfigRepository, dockerReg dockerRegistryRepository.DockerArtifactStoreRepository, attributeRepo repository.AttributesRepository, gitProviderRepository repository.GitProviderRepository, @@ -62,7 +60,6 @@ func NewCommonServiceImpl(logger *zap.SugaredLogger, logger: logger, chartRepository: chartRepository, environmentConfigRepository: environmentConfigRepository, - gitOpsRepository: gitOpsRepository, dockerReg: dockerReg, attributeRepo: attributeRepo, gitProviderRepository: gitProviderRepository, diff --git a/pkg/deployment/gitOps/common/bean/bean.go b/pkg/deployment/gitOps/common/bean/bean.go new file mode 100644 index 0000000000..4241070a37 --- /dev/null +++ b/pkg/deployment/gitOps/common/bean/bean.go @@ -0,0 +1,6 @@ +package bean + +// TODO : rename +type ChartGitAttribute struct { + RepoUrl, ChartLocation string +} diff --git a/pkg/deployment/gitOps/config/GitOpsConfigReadService.go b/pkg/deployment/gitOps/config/GitOpsConfigReadService.go new file mode 100644 index 0000000000..6daf96c6d3 --- /dev/null +++ b/pkg/deployment/gitOps/config/GitOpsConfigReadService.go @@ -0,0 +1,133 @@ +package config + +import ( + "fmt" + bean2 "github.com/devtron-labs/devtron/api/bean" + "github.com/devtron-labs/devtron/internal/sql/repository" + "github.com/devtron-labs/devtron/pkg/auth/user" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config/bean" + "github.com/devtron-labs/devtron/util" + "github.com/go-pg/pg" + "go.uber.org/zap" + "regexp" + "strings" +) + +type GitOpsConfigReadService interface { + IsGitOpsConfigured() (bool, error) + GetUserEmailIdAndNameForGitOpsCommit(userId int32) (string, string) + GetGitOpsRepoName(appName string) string + GetGitOpsRepoNameFromUrl(gitRepoUrl string) string + GetBitbucketMetadata() (*bean.BitbucketProviderMetadata, error) + GetGitOpsConfigActive() (*bean2.GitOpsConfigDto, error) +} + +type GitOpsConfigReadServiceImpl struct { + logger *zap.SugaredLogger + gitOpsRepository repository.GitOpsConfigRepository + userService user.UserService + globalEnvVariables *util.GlobalEnvVariables +} + +func NewGitOpsConfigReadServiceImpl(logger *zap.SugaredLogger, + gitOpsRepository repository.GitOpsConfigRepository, + userService user.UserService, + globalEnvVariables *util.GlobalEnvVariables) *GitOpsConfigReadServiceImpl { + return &GitOpsConfigReadServiceImpl{ + logger: logger, + gitOpsRepository: gitOpsRepository, + userService: userService, + globalEnvVariables: globalEnvVariables, + } +} + +func (impl *GitOpsConfigReadServiceImpl) IsGitOpsConfigured() (bool, error) { + isGitOpsConfigured := false + gitOpsConfig, err := impl.gitOpsRepository.GetGitOpsConfigActive() + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("GetGitOpsConfigActive, error while getting", "err", err) + return false, err + } + if gitOpsConfig != nil && gitOpsConfig.Id > 0 { + isGitOpsConfigured = true + } + return isGitOpsConfigured, nil +} + +func (impl *GitOpsConfigReadServiceImpl) GetUserEmailIdAndNameForGitOpsCommit(userId int32) (string, string) { + emailId := bean.GitOpsCommitDefaultEmailId + name := bean.GitOpsCommitDefaultName + //getting emailId associated with user + userEmail, err := impl.userService.GetEmailById(userId) + if err != nil { + impl.logger.Errorw("error in getting user info by id", "err", err, "id", userId) + } + //TODO: export constant in user bean + if userEmail != "admin" && userEmail != "system" && len(userEmail) > 0 { + emailId = userEmail + } else { + emailIdGitOps, err := impl.gitOpsRepository.GetEmailIdFromActiveGitOpsConfig() + if err != nil { + impl.logger.Errorw("error in getting emailId from active gitOps config", "err", err) + } else if len(emailIdGitOps) > 0 { + emailId = emailIdGitOps + } + } + //we are getting name from emailId(replacing special characters in with space) + emailComponents := strings.Split(emailId, "@") + regex, _ := regexp.Compile(`[^\w]`) + if regex != nil { + name = regex.ReplaceAllString(emailComponents[0], " ") + } + return emailId, name +} + +func (impl *GitOpsConfigReadServiceImpl) GetGitOpsRepoName(appName string) string { + var repoName string + if len(impl.globalEnvVariables.GitOpsRepoPrefix) == 0 { + repoName = appName + } else { + repoName = fmt.Sprintf("%s-%s", impl.globalEnvVariables.GitOpsRepoPrefix, appName) + } + return repoName +} + +func (impl *GitOpsConfigReadServiceImpl) GetGitOpsRepoNameFromUrl(gitRepoUrl string) string { + gitRepoUrl = gitRepoUrl[strings.LastIndex(gitRepoUrl, "/")+1:] + gitRepoUrl = strings.ReplaceAll(gitRepoUrl, ".git", "") + return gitRepoUrl +} + +func (impl *GitOpsConfigReadServiceImpl) GetBitbucketMetadata() (*bean.BitbucketProviderMetadata, error) { + metadata := &bean.BitbucketProviderMetadata{} + gitOpsConfigBitbucket, err := impl.gitOpsRepository.GetGitOpsConfigByProvider(bean.BITBUCKET_PROVIDER) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in fetching gitOps bitbucket config", "err", err) + return nil, err + } + if gitOpsConfigBitbucket != nil { + metadata.BitBucketWorkspaceId = gitOpsConfigBitbucket.BitBucketWorkspaceId + metadata.BitBucketProjectKey = gitOpsConfigBitbucket.BitBucketProjectKey + } + return metadata, nil +} + +func (impl *GitOpsConfigReadServiceImpl) GetGitOpsConfigActive() (*bean2.GitOpsConfigDto, error) { + model, err := impl.gitOpsRepository.GetGitOpsConfigActive() + if err != nil { + impl.logger.Errorw("error, GetGitOpsConfigActive", "err", err) + return nil, err + } + config := &bean2.GitOpsConfigDto{ + Id: model.Id, + Provider: model.Provider, + GitHubOrgId: model.GitHubOrgId, + GitLabGroupId: model.GitLabGroupId, + Active: model.Active, + UserId: model.CreatedBy, + AzureProjectName: model.AzureProject, + BitBucketWorkspaceId: model.BitBucketWorkspaceId, + BitBucketProjectKey: model.BitBucketProjectKey, + } + return config, err +} diff --git a/pkg/deployment/gitOps/config/bean/bean.go b/pkg/deployment/gitOps/config/bean/bean.go new file mode 100644 index 0000000000..8e6a43a51c --- /dev/null +++ b/pkg/deployment/gitOps/config/bean/bean.go @@ -0,0 +1,14 @@ +package bean + +const ( + GitOpsCommitDefaultEmailId = "devtron-bot@devtron.ai" + GitOpsCommitDefaultName = "devtron bot" +) + +// TODO: remove below object and its related methods to eliminate provider specific signature +type BitbucketProviderMetadata struct { + BitBucketWorkspaceId string + BitBucketProjectKey string +} + +const BITBUCKET_PROVIDER = "BITBUCKET_CLOUD" diff --git a/pkg/deployment/gitOps/remote/GitOpsRemoteOperationService.go b/pkg/deployment/gitOps/remote/GitOpsRemoteOperationService.go new file mode 100644 index 0000000000..f956e862e8 --- /dev/null +++ b/pkg/deployment/gitOps/remote/GitOpsRemoteOperationService.go @@ -0,0 +1,316 @@ +package remote + +import ( + "fmt" + bean2 "github.com/devtron-labs/devtron/api/bean" + "github.com/devtron-labs/devtron/internal/util" + commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote/bean" + dirCopy "github.com/otiai10/copy" + "go.uber.org/zap" + "k8s.io/helm/pkg/proto/hapi/chart" + "os" + "path/filepath" + "regexp" + "sigs.k8s.io/yaml" +) + +type GitOpsRemoteOperationService interface { + CreateGitRepositoryForApp(gitOpsRepoName, baseTemplateName, + version string, userId int32) (chartGitAttribute *commonBean.ChartGitAttribute, err error) + PushChartToGitRepo(gitOpsRepoName, referenceTemplate, version, + tempReferenceTemplateDir string, repoUrl string, userId int32) (err error) + CreateReadmeInGitRepo(gitOpsRepoName string, userId int32) error + CreateChartProxy(chartMetaData *chart.Metadata, refChartLocation string, envName string, + chartProxyReq *bean.ChartProxyReqDto) (string, *commonBean.ChartGitAttribute, error) + GitPull(clonedDir string, repoUrl string, appStoreName string) error +} + +type GitOpsRemoteOperationServiceImpl struct { + logger *zap.SugaredLogger + gitFactory *util.GitFactory + gitOpsConfigReadService config.GitOpsConfigReadService + chartTemplateService util.ChartTemplateService +} + +func NewGitOpsRemoteOperationServiceImpl(logger *zap.SugaredLogger, gitFactory *util.GitFactory, + gitOpsConfigReadService config.GitOpsConfigReadService, + chartTemplateService util.ChartTemplateService) *GitOpsRemoteOperationServiceImpl { + return &GitOpsRemoteOperationServiceImpl{ + logger: logger, + gitOpsConfigReadService: gitOpsConfigReadService, + chartTemplateService: chartTemplateService, + } + +} + +func (impl *GitOpsRemoteOperationServiceImpl) CreateGitRepositoryForApp(gitOpsRepoName, baseTemplateName, + version string, userId int32) (chartGitAttribute *commonBean.ChartGitAttribute, err error) { + //baseTemplateName replace whitespace + space := regexp.MustCompile(`\s+`) + gitOpsRepoName = space.ReplaceAllString(gitOpsRepoName, "-") + + bitbucketMetadata, err := impl.gitOpsConfigReadService.GetBitbucketMetadata() + if err != nil { + impl.logger.Errorw("error in getting bitbucket metadata", "err", err) + return nil, err + } + //getting user name & emailId for commit author data + userEmailId, userName := impl.gitOpsConfigReadService.GetUserEmailIdAndNameForGitOpsCommit(userId) + gitRepoRequest := &bean2.GitOpsConfigDto{ + GitRepoName: gitOpsRepoName, + Description: fmt.Sprintf("helm chart for " + gitOpsRepoName), + Username: userName, + UserEmailId: userEmailId, + BitBucketWorkspaceId: bitbucketMetadata.BitBucketWorkspaceId, + BitBucketProjectKey: bitbucketMetadata.BitBucketProjectKey, + } + repoUrl, _, detailedError := impl.gitFactory.Client.CreateRepository(gitRepoRequest) + for _, err := range detailedError.StageErrorMap { + if err != nil { + impl.logger.Errorw("error in creating git project", "name", gitOpsRepoName, "err", err) + return nil, err + } + } + return &commonBean.ChartGitAttribute{RepoUrl: repoUrl, ChartLocation: filepath.Join(baseTemplateName, version)}, nil +} + +func (impl *GitOpsRemoteOperationServiceImpl) PushChartToGitRepo(gitOpsRepoName, referenceTemplate, version, + tempReferenceTemplateDir string, repoUrl string, userId int32) (err error) { + chartDir := fmt.Sprintf("%s-%s", gitOpsRepoName, impl.chartTemplateService.GetDir()) + clonedDir := impl.gitFactory.GitService.GetCloneDirectory(chartDir) + if _, err := os.Stat(clonedDir); os.IsNotExist(err) { + clonedDir, err = impl.gitFactory.GitService.Clone(repoUrl, chartDir) + if err != nil { + impl.logger.Errorw("error in cloning repo", "url", repoUrl, "err", err) + return err + } + } else { + err = impl.GitPull(clonedDir, repoUrl, gitOpsRepoName) + if err != nil { + impl.logger.Errorw("error in pulling git repo", "url", repoUrl, "err", err) + return err + } + } + + dir := filepath.Join(clonedDir, referenceTemplate, version) + pushChartToGit := true + + //if chart already exists don't overrides it by reference template + if _, err := os.Stat(dir); os.IsNotExist(err) { + err = os.MkdirAll(dir, os.ModePerm) + if err != nil { + impl.logger.Errorw("error in making dir", "err", err) + return err + } + err = dirCopy.Copy(tempReferenceTemplateDir, dir) + if err != nil { + impl.logger.Errorw("error copying dir", "err", err) + return err + } + } else { + // auto-healing : data corruption fix - sometimes reference chart contents are not pushed in git-ops repo. + // copying content from reference template dir to cloned dir (if Chart.yaml file is not found) + // if Chart.yaml file is not found, we are assuming here that reference chart contents are not pushed in git-ops repo + if _, err := os.Stat(filepath.Join(dir, "Chart.yaml")); os.IsNotExist(err) { + impl.logger.Infow("auto-healing: Chart.yaml not found in cloned repo from git-ops. copying content", "from", tempReferenceTemplateDir, "to", dir) + err = dirCopy.Copy(tempReferenceTemplateDir, dir) + if err != nil { + impl.logger.Errorw("error copying content in auto-healing", "err", err) + return err + } + } else { + // chart exists on git, hence not performing first commit + pushChartToGit = false + } + } + + // if push needed, then only push + if pushChartToGit { + userEmailId, userName := impl.gitOpsConfigReadService.GetUserEmailIdAndNameForGitOpsCommit(userId) + commit, err := impl.gitFactory.GitService.CommitAndPushAllChanges(clonedDir, "first commit", userName, userEmailId) + if err != nil { + impl.logger.Errorw("error in pushing git", "err", err) + impl.logger.Warn("re-trying, taking pull and then push again") + err = impl.GitPull(clonedDir, repoUrl, gitOpsRepoName) + if err != nil { + return err + } + err = dirCopy.Copy(tempReferenceTemplateDir, dir) + if err != nil { + impl.logger.Errorw("error copying dir", "err", err) + return err + } + commit, err = impl.gitFactory.GitService.CommitAndPushAllChanges(clonedDir, "first commit", userName, userEmailId) + if err != nil { + impl.logger.Errorw("error in pushing git", "err", err) + return err + } + } + impl.logger.Debugw("template committed", "url", repoUrl, "commit", commit) + } + + defer impl.chartTemplateService.CleanDir(clonedDir) + return nil +} + +func (impl *GitOpsRemoteOperationServiceImpl) CreateReadmeInGitRepo(gitOpsRepoName string, userId int32) error { + userEmailId, userName := impl.gitOpsConfigReadService.GetUserEmailIdAndNameForGitOpsCommit(userId) + gitOpsConfig, err := impl.gitOpsConfigReadService.GetGitOpsConfigActive() + if err != nil { + impl.logger.Errorw("error in getting active gitOps config", "err", err) + return err + } + //updating user email and name in request + if gitOpsConfig != nil { + gitOpsConfig.UserEmailId = userEmailId + gitOpsConfig.Username = userName + gitOpsConfig.GitRepoName = gitOpsRepoName + } + _, err = impl.gitFactory.Client.CreateReadme(gitOpsConfig) + if err != nil { + return err + } + return nil +} + +func (impl *GitOpsRemoteOperationServiceImpl) CreateChartProxy(chartMetaData *chart.Metadata, refChartLocation string, envName string, + chartProxyReq *bean.ChartProxyReqDto) (string, *commonBean.ChartGitAttribute, error) { + chartMetaData.ApiVersion = "v2" // ensure always v2 + dir := impl.chartTemplateService.GetDir() + chartDir := filepath.Join(util.ChartWorkingDirPath, dir) + impl.logger.Debugw("chart dir ", "chart", chartMetaData.Name, "dir", chartDir) + err := os.MkdirAll(chartDir, os.ModePerm) //hack for concurrency handling + if err != nil { + impl.logger.Errorw("err in creating dir", "dir", chartDir, "err", err) + return "", nil, err + } + defer impl.chartTemplateService.CleanDir(chartDir) + err = dirCopy.Copy(refChartLocation, chartDir) + + if err != nil { + impl.logger.Errorw("error in copying chart for app", "app", chartMetaData.Name, "error", err) + return "", nil, err + } + archivePath, valuesYaml, err := impl.chartTemplateService.PackageChart(chartDir, chartMetaData) + if err != nil { + impl.logger.Errorw("error in creating archive", "err", err) + return "", nil, err + } + + chartGitAttr, err := impl.createAndPushToGitChartProxy(chartMetaData.Name, chartDir, envName, chartProxyReq) + if err != nil { + impl.logger.Errorw("error in pushing chart to git ", "path", archivePath, "err", err) + return "", nil, err + } + if valuesYaml == "" { + valuesYaml = "{}" + } else { + valuesYamlByte, err := yaml.YAMLToJSON([]byte(valuesYaml)) + if err != nil { + return "", nil, err + } + valuesYaml = string(valuesYamlByte) + } + return valuesYaml, chartGitAttr, nil +} + +func (impl *GitOpsRemoteOperationServiceImpl) createAndPushToGitChartProxy(appStoreName, tmpChartLocation string, envName string, + chartProxyReq *bean.ChartProxyReqDto) (chartGitAttribute *commonBean.ChartGitAttribute, err error) { + //baseTemplateName replace whitespace + space := regexp.MustCompile(`\s+`) + appStoreName = space.ReplaceAllString(appStoreName, "-") + + if len(chartProxyReq.GitOpsRepoName) == 0 { + //here git ops repo will be the app name, to breaking the mono repo structure + gitOpsRepoName := impl.gitOpsConfigReadService.GetGitOpsRepoName(chartProxyReq.AppName) + chartProxyReq.GitOpsRepoName = gitOpsRepoName + } + bitbucketMetadata, err := impl.gitOpsConfigReadService.GetBitbucketMetadata() + if err != nil { + impl.logger.Errorw("error in getting bitbucket metadata", "err", err) + return nil, err + } + //getting user name & emailId for commit author data + userEmailId, userName := impl.gitOpsConfigReadService.GetUserEmailIdAndNameForGitOpsCommit(chartProxyReq.UserId) + gitRepoRequest := &bean2.GitOpsConfigDto{ + GitRepoName: chartProxyReq.GitOpsRepoName, + Description: "helm chart for " + chartProxyReq.GitOpsRepoName, + Username: userName, + UserEmailId: userEmailId, + BitBucketWorkspaceId: bitbucketMetadata.BitBucketWorkspaceId, + BitBucketProjectKey: bitbucketMetadata.BitBucketProjectKey, + } + repoUrl, _, detailedError := impl.gitFactory.Client.CreateRepository(gitRepoRequest) + for _, err := range detailedError.StageErrorMap { + if err != nil { + impl.logger.Errorw("error in creating git project", "name", chartProxyReq.GitOpsRepoName, "err", err) + return nil, err + } + } + + chartDir := fmt.Sprintf("%s-%s", chartProxyReq.AppName, impl.chartTemplateService.GetDir()) + clonedDir := impl.gitFactory.GitService.GetCloneDirectory(chartDir) + if _, err := os.Stat(clonedDir); os.IsNotExist(err) { + clonedDir, err = impl.gitFactory.GitService.Clone(repoUrl, chartDir) + if err != nil { + impl.logger.Errorw("error in cloning repo", "url", repoUrl, "err", err) + return nil, err + } + } else { + err = impl.GitPull(clonedDir, repoUrl, appStoreName) + if err != nil { + return nil, err + } + } + + acdAppName := fmt.Sprintf("%s-%s", chartProxyReq.AppName, envName) + dir := filepath.Join(clonedDir, acdAppName) + err = os.MkdirAll(dir, os.ModePerm) + if err != nil { + impl.logger.Errorw("error in making dir", "err", err) + return nil, err + } + err = dirCopy.Copy(tmpChartLocation, dir) + if err != nil { + impl.logger.Errorw("error copying dir", "err", err) + return nil, err + } + commit, err := impl.gitFactory.GitService.CommitAndPushAllChanges(clonedDir, "first commit", userName, userEmailId) + if err != nil { + impl.logger.Errorw("error in pushing git", "err", err) + impl.logger.Warn("re-trying, taking pull and then push again") + err = impl.GitPull(clonedDir, repoUrl, acdAppName) + if err != nil { + return nil, err + } + err = dirCopy.Copy(tmpChartLocation, dir) + if err != nil { + impl.logger.Errorw("error copying dir", "err", err) + return nil, err + } + commit, err = impl.gitFactory.GitService.CommitAndPushAllChanges(clonedDir, "first commit", userName, userEmailId) + if err != nil { + impl.logger.Errorw("error in pushing git", "err", err) + return nil, err + } + } + impl.logger.Debugw("template committed", "url", repoUrl, "commit", commit) + defer impl.chartTemplateService.CleanDir(clonedDir) + return &commonBean.ChartGitAttribute{RepoUrl: repoUrl, ChartLocation: filepath.Join("", acdAppName)}, nil +} + +func (impl *GitOpsRemoteOperationServiceImpl) GitPull(clonedDir string, repoUrl string, appStoreName string) error { + err := impl.gitFactory.GitService.Pull(clonedDir) //TODO check for local repo exists before clone + if err != nil { + impl.logger.Errorw("error in pulling git", "clonedDir", clonedDir, "err", err) + _, err := impl.gitFactory.GitService.Clone(repoUrl, appStoreName) + if err != nil { + impl.logger.Errorw("error in cloning repo", "url", repoUrl, "err", err) + return err + } + return nil + } + return nil +} diff --git a/pkg/deployment/gitOps/remote/bean/bean.go b/pkg/deployment/gitOps/remote/bean/bean.go new file mode 100644 index 0000000000..361b0ec78c --- /dev/null +++ b/pkg/deployment/gitOps/remote/bean/bean.go @@ -0,0 +1,7 @@ +package bean + +type ChartProxyReqDto struct { + GitOpsRepoName string `json:"gitOpsRepoName"` + AppName string `json:"appName,omitempty"` + UserId int32 `json:"-"` +} diff --git a/pkg/deployment/gitOps/wire_gitOps.go b/pkg/deployment/gitOps/wire_gitOps.go new file mode 100644 index 0000000000..7d5378d238 --- /dev/null +++ b/pkg/deployment/gitOps/wire_gitOps.go @@ -0,0 +1,15 @@ +package gitOps + +import ( + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote" + "github.com/google/wire" +) + +var GitOpsWireSet = wire.NewSet( + config.NewGitOpsConfigReadServiceImpl, + wire.Bind(new(config.GitOpsConfigReadService), new(*config.GitOpsConfigReadServiceImpl)), + + remote.NewGitOpsRemoteOperationServiceImpl, + wire.Bind(new(remote.GitOpsRemoteOperationService), new(*remote.GitOpsRemoteOperationServiceImpl)), +) diff --git a/pkg/deployment/wire_deployment.go b/pkg/deployment/wire_deployment.go new file mode 100644 index 0000000000..5a959cd701 --- /dev/null +++ b/pkg/deployment/wire_deployment.go @@ -0,0 +1,14 @@ +package deployment + +import ( + "github.com/devtron-labs/devtron/pkg/deployment/gitOps" + "github.com/devtron-labs/devtron/pkg/deployment/manifest" + "github.com/google/wire" +) + +// TODO: add separate wire sets for full and ea mode when reached that level of transparency + +var DeploymentWireSet = wire.NewSet( + gitOps.GitOpsWireSet, + manifest.DeploymentManifestWireSet, +) diff --git a/pkg/gitops/GitOpsConfigService.go b/pkg/gitops/GitOpsConfigService.go index ed3c6bd27b..3b295f3a65 100644 --- a/pkg/gitops/GitOpsConfigService.go +++ b/pkg/gitops/GitOpsConfigService.go @@ -22,6 +22,8 @@ import ( "encoding/json" "fmt" util4 "github.com/devtron-labs/common-lib/utils/k8s" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config/bean" "math/rand" "net/http" "net/url" @@ -58,23 +60,22 @@ type GitOpsConfigService interface { GetGitOpsConfigById(id int) (*bean2.GitOpsConfigDto, error) GetAllGitOpsConfig() ([]*bean2.GitOpsConfigDto, error) GetGitOpsConfigByProvider(provider string) (*bean2.GitOpsConfigDto, error) - GetGitOpsConfigActive() (*bean2.GitOpsConfigDto, error) } const ( - GitOpsSecretName = "devtron-gitops-secret" - DryrunRepoName = "devtron-sample-repo-dryrun-" - DeleteRepoStage = "Delete Repo" - CommitOnRestStage = "Commit On Rest" - PushStage = "Push" - CloneStage = "Clone" - GetRepoUrlStage = "Get Repo RedirectionUrl" - CreateRepoStage = "Create Repo" - CloneHttp = "Clone Http" - CreateReadmeStage = "Create Readme" - GITHUB_PROVIDER = "GITHUB" - GITLAB_PROVIDER = "GITLAB" - BITBUCKET_PROVIDER = "BITBUCKET_CLOUD" + GitOpsSecretName = "devtron-gitops-secret" + DryrunRepoName = "devtron-sample-repo-dryrun-" + DeleteRepoStage = "Delete Repo" + CommitOnRestStage = "Commit On Rest" + PushStage = "Push" + CloneStage = "Clone" + GetRepoUrlStage = "Get Repo RedirectionUrl" + CreateRepoStage = "Create Repo" + CloneHttp = "Clone Http" + CreateReadmeStage = "Create Readme" + GITHUB_PROVIDER = "GITHUB" + GITLAB_PROVIDER = "GITLAB" + AZURE_DEVOPS_PROVIDER = "AZURE_DEVOPS" BITBUCKET_API_HOST = "https://api.bitbucket.org/2.0/" ) @@ -87,36 +88,37 @@ type DetailedErrorGitOpsConfigResponse struct { } type GitOpsConfigServiceImpl struct { - randSource rand.Source - logger *zap.SugaredLogger - globalEnvVariables *util2.GlobalEnvVariables - gitOpsRepository repository.GitOpsConfigRepository - K8sUtil *util4.K8sUtil - aCDAuthConfig *util3.ACDAuthConfig - clusterService cluster.ClusterService - gitFactory *util.GitFactory - chartTemplateService util.ChartTemplateService - argoUserService argo.ArgoUserService - clusterServiceCD cluster2.ServiceClient + randSource rand.Source + logger *zap.SugaredLogger + globalEnvVariables *util2.GlobalEnvVariables + gitOpsRepository repository.GitOpsConfigRepository + K8sUtil *util4.K8sUtil + aCDAuthConfig *util3.ACDAuthConfig + clusterService cluster.ClusterService + gitFactory *util.GitFactory + argoUserService argo.ArgoUserService + clusterServiceCD cluster2.ServiceClient + gitOpsConfigReadService config.GitOpsConfigReadService } func NewGitOpsConfigServiceImpl(Logger *zap.SugaredLogger, globalEnvVariables *util2.GlobalEnvVariables, gitOpsRepository repository.GitOpsConfigRepository, K8sUtil *util4.K8sUtil, aCDAuthConfig *util3.ACDAuthConfig, clusterService cluster.ClusterService, - gitFactory *util.GitFactory, chartTemplateService util.ChartTemplateService, argoUserService argo.ArgoUserService, clusterServiceCD cluster2.ServiceClient) *GitOpsConfigServiceImpl { + gitFactory *util.GitFactory, argoUserService argo.ArgoUserService, + clusterServiceCD cluster2.ServiceClient, gitOpsConfigReadService config.GitOpsConfigReadService) *GitOpsConfigServiceImpl { return &GitOpsConfigServiceImpl{ - randSource: rand.NewSource(time.Now().UnixNano()), - logger: Logger, - globalEnvVariables: globalEnvVariables, - gitOpsRepository: gitOpsRepository, - K8sUtil: K8sUtil, - aCDAuthConfig: aCDAuthConfig, - clusterService: clusterService, - gitFactory: gitFactory, - chartTemplateService: chartTemplateService, - argoUserService: argoUserService, - clusterServiceCD: clusterServiceCD, + randSource: rand.NewSource(time.Now().UnixNano()), + logger: Logger, + globalEnvVariables: globalEnvVariables, + gitOpsRepository: gitOpsRepository, + K8sUtil: K8sUtil, + aCDAuthConfig: aCDAuthConfig, + clusterService: clusterService, + gitFactory: gitFactory, + argoUserService: argoUserService, + clusterServiceCD: clusterServiceCD, + gitOpsConfigReadService: gitOpsConfigReadService, } } @@ -287,7 +289,7 @@ func (impl *GitOpsConfigServiceImpl) CreateGitOpsConfig(ctx context.Context, req request.Host = fmt.Sprintf(request.Host+"/%s", groupName) } } - if strings.ToUpper(request.Provider) == BITBUCKET_PROVIDER { + if strings.ToUpper(request.Provider) == bean.BITBUCKET_PROVIDER { request.Host = util.BITBUCKET_CLONE_BASE_URL + request.BitBucketWorkspaceId } operationComplete := false @@ -488,7 +490,7 @@ func (impl *GitOpsConfigServiceImpl) UpdateGitOpsConfig(request *bean2.GitOpsCon request.Host = fmt.Sprintf(request.Host+"/%s", groupName) } } - if strings.ToUpper(request.Provider) == BITBUCKET_PROVIDER { + if strings.ToUpper(request.Provider) == bean.BITBUCKET_PROVIDER { request.Host = util.BITBUCKET_CLONE_BASE_URL + request.BitBucketWorkspaceId } operationComplete := false @@ -661,26 +663,6 @@ type KeyDto struct { Key string `json:"key,omitempty"` } -func (impl *GitOpsConfigServiceImpl) GetGitOpsConfigActive() (*bean2.GitOpsConfigDto, error) { - model, err := impl.gitOpsRepository.GetGitOpsConfigActive() - if err != nil { - impl.logger.Errorw("GetGitOpsConfigActive, error while getting error", "err", err) - return nil, err - } - config := &bean2.GitOpsConfigDto{ - Id: model.Id, - Provider: model.Provider, - GitHubOrgId: model.GitHubOrgId, - GitLabGroupId: model.GitLabGroupId, - Active: model.Active, - UserId: model.CreatedBy, - AzureProjectName: model.AzureProject, - BitBucketWorkspaceId: model.BitBucketWorkspaceId, - BitBucketProjectKey: model.BitBucketProjectKey, - } - return config, err -} - func (impl *GitOpsConfigServiceImpl) GitOpsValidateDryRun(config *bean2.GitOpsConfigDto) DetailedErrorGitOpsConfigResponse { if impl.globalEnvVariables.SkipGitOpsValidation { return DetailedErrorGitOpsConfigResponse{} @@ -702,7 +684,7 @@ func (impl *GitOpsConfigServiceImpl) GitOpsValidateDryRun(config *bean2.GitOpsCo /*if strings.ToUpper(config.Provider) == GITHUB_PROVIDER { config.Host = GITHUB_HOST }*/ - if strings.ToUpper(config.Provider) == BITBUCKET_PROVIDER { + if strings.ToUpper(config.Provider) == bean.BITBUCKET_PROVIDER { config.Host = util.BITBUCKET_CLONE_BASE_URL config.BitBucketProjectKey = strings.ToUpper(config.BitBucketProjectKey) } @@ -716,7 +698,7 @@ func (impl *GitOpsConfigServiceImpl) GitOpsValidateDryRun(config *bean2.GitOpsCo } appName := DryrunRepoName + util2.Generate(6) //getting user name & emailId for commit author data - userEmailId, userName := impl.chartTemplateService.GetUserEmailIdAndNameForGitOpsCommit(config.UserId) + userEmailId, userName := impl.gitOpsConfigReadService.GetUserEmailIdAndNameForGitOpsCommit(config.UserId) config.UserEmailId = userEmailId config.GitRepoName = appName repoUrl, _, detailedErrorCreateRepo := client.CreateRepository(config) diff --git a/pkg/pipeline/AppDeploymentTypeChangeManager.go b/pkg/pipeline/AppDeploymentTypeChangeManager.go index 4e12e27092..5f340c755f 100644 --- a/pkg/pipeline/AppDeploymentTypeChangeManager.go +++ b/pkg/pipeline/AppDeploymentTypeChangeManager.go @@ -29,6 +29,8 @@ import ( "github.com/devtron-labs/devtron/internal/util" app2 "github.com/devtron-labs/devtron/pkg/app" "github.com/devtron-labs/devtron/pkg/bean" + chartService "github.com/devtron-labs/devtron/pkg/chart" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" "github.com/juju/errors" "go.uber.org/zap" "strconv" @@ -55,17 +57,18 @@ type AppDeploymentTypeChangeManager interface { } type AppDeploymentTypeChangeManagerImpl struct { - logger *zap.SugaredLogger - pipelineRepository pipelineConfig.PipelineRepository - workflowDagExecutor WorkflowDagExecutor - appService app2.AppService - chartTemplateService util.ChartTemplateService - appStatusRepository appStatus.AppStatusRepository - helmAppService client.HelmAppService - application application2.ServiceClient + logger *zap.SugaredLogger + pipelineRepository pipelineConfig.PipelineRepository + workflowDagExecutor WorkflowDagExecutor + appService app2.AppService + appStatusRepository appStatus.AppStatusRepository + helmAppService client.HelmAppService + application application2.ServiceClient appArtifactManager AppArtifactManager cdPipelineConfigService CdPipelineConfigService + gitOpsConfigReadService config.GitOpsConfigReadService + chartService chartService.ChartService } func NewAppDeploymentTypeChangeManagerImpl( @@ -73,23 +76,25 @@ func NewAppDeploymentTypeChangeManagerImpl( pipelineRepository pipelineConfig.PipelineRepository, workflowDagExecutor WorkflowDagExecutor, appService app2.AppService, - chartTemplateService util.ChartTemplateService, appStatusRepository appStatus.AppStatusRepository, helmAppService client.HelmAppService, application application2.ServiceClient, appArtifactManager AppArtifactManager, - cdPipelineConfigService CdPipelineConfigService) *AppDeploymentTypeChangeManagerImpl { + cdPipelineConfigService CdPipelineConfigService, + gitOpsConfigReadService config.GitOpsConfigReadService, + chartService chartService.ChartService) *AppDeploymentTypeChangeManagerImpl { return &AppDeploymentTypeChangeManagerImpl{ logger: logger, pipelineRepository: pipelineRepository, workflowDagExecutor: workflowDagExecutor, appService: appService, - chartTemplateService: chartTemplateService, appStatusRepository: appStatusRepository, helmAppService: helmAppService, application: application, appArtifactManager: appArtifactManager, cdPipelineConfigService: cdPipelineConfigService, + gitOpsConfigReadService: gitOpsConfigReadService, + chartService: chartService, } } @@ -397,7 +402,7 @@ func (impl *AppDeploymentTypeChangeManagerImpl) DeleteDeploymentApps(ctx context successfulPipelines := make([]*bean.DeploymentChangeStatus, 0) failedPipelines := make([]*bean.DeploymentChangeStatus, 0) - isGitOpsConfigured, gitOpsConfigErr := impl.cdPipelineConfigService.IsGitopsConfigured() + isGitOpsConfigured, gitOpsConfigErr := impl.gitOpsConfigReadService.IsGitOpsConfigured() // Iterate over all the pipelines in the environment for given deployment app type for _, pipeline := range pipelines { @@ -445,7 +450,7 @@ func (impl *AppDeploymentTypeChangeManagerImpl) DeleteDeploymentApps(ctx context impl.logger.Errorw("error in registering acd app", "err", err) } if AcdRegisterErr == nil { - RepoURLUpdateErr = impl.chartTemplateService.UpdateGitRepoUrlInCharts(pipeline.AppId, chartGitAttr, userId) + RepoURLUpdateErr = impl.chartService.UpdateGitRepoUrlInCharts(pipeline.AppId, chartGitAttr.RepoUrl, chartGitAttr.ChartLocation, userId) if RepoURLUpdateErr != nil { impl.logger.Errorw("error in updating git repo url in charts", "err", err) } diff --git a/pkg/pipeline/DeploymentPipelineConfigService.go b/pkg/pipeline/DeploymentPipelineConfigService.go index 794f7bfaac..98e9c35512 100644 --- a/pkg/pipeline/DeploymentPipelineConfigService.go +++ b/pkg/pipeline/DeploymentPipelineConfigService.go @@ -41,6 +41,9 @@ import ( chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" "github.com/devtron-labs/devtron/pkg/cluster" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" + commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/history" @@ -109,8 +112,7 @@ type CdPipelineConfigService interface { MarkGitOpsDevtronAppsDeletedWhereArgoAppIsDeleted(appId int, envId int, acdToken string, pipeline *pipelineConfig.Pipeline) (bool, error) //GetEnvironmentListForAutocompleteFilter : lists environment for given configuration GetEnvironmentListForAutocompleteFilter(envName string, clusterIds []int, offset int, size int, token string, checkAuthBatch func(token string, appObject []string, envObject []string) (map[string]bool, map[string]bool), ctx context.Context) (*cluster.ResourceGroupingResponse, error) - IsGitopsConfigured() (bool, error) - RegisterInACD(gitOpsRepoName string, chartGitAttr *util.ChartGitAttribute, userId int32, ctx context.Context) error + RegisterInACD(gitOpsRepoName string, chartGitAttr *commonBean.ChartGitAttribute, userId int32, ctx context.Context) error } type CdPipelineConfigServiceImpl struct { @@ -130,11 +132,9 @@ type CdPipelineConfigServiceImpl struct { clusterRepository repository2.ClusterRepository helmAppService client.HelmAppService enforcerUtil rbac.EnforcerUtil - gitOpsRepository repository.GitOpsConfigRepository pipelineStrategyHistoryService history.PipelineStrategyHistoryService chartRepository chartRepoRepository.ChartRepository resourceGroupService resourceGroup2.ResourceGroupService - chartTemplateService util.ChartTemplateService propertiesConfigService PropertiesConfigService deploymentTemplateHistoryService history.DeploymentTemplateHistoryService scopedVariableManager variables.ScopedVariableManager @@ -147,6 +147,8 @@ type CdPipelineConfigServiceImpl struct { buildPipelineSwitchService BuildPipelineSwitchService argoClientWrapperService argocdServer.ArgoClientWrapperService deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService + gitOpsConfigReadService config.GitOpsConfigReadService + gitOpsRemoteOperationService remote.GitOpsRemoteOperationService } func NewCdPipelineConfigServiceImpl(logger *zap.SugaredLogger, pipelineRepository pipelineConfig.PipelineRepository, @@ -156,17 +158,18 @@ func NewCdPipelineConfigServiceImpl(logger *zap.SugaredLogger, pipelineRepositor ciCdPipelineOrchestrator CiCdPipelineOrchestrator, appStatusRepository appStatus.AppStatusRepository, ciPipelineRepository pipelineConfig.CiPipelineRepository, prePostCdScriptHistoryService history.PrePostCdScriptHistoryService, clusterRepository repository2.ClusterRepository, helmAppService client.HelmAppService, - enforcerUtil rbac.EnforcerUtil, gitOpsRepository repository.GitOpsConfigRepository, - pipelineStrategyHistoryService history.PipelineStrategyHistoryService, + enforcerUtil rbac.EnforcerUtil, pipelineStrategyHistoryService history.PipelineStrategyHistoryService, chartRepository chartRepoRepository.ChartRepository, resourceGroupService resourceGroup2.ResourceGroupService, - chartTemplateService util.ChartTemplateService, propertiesConfigService PropertiesConfigService, + propertiesConfigService PropertiesConfigService, deploymentTemplateHistoryService history.DeploymentTemplateHistoryService, scopedVariableManager variables.ScopedVariableManager, deploymentConfig *DeploymentServiceTypeConfig, application application.ServiceClient, customTagService CustomTagService, pipelineConfigListenerService PipelineConfigListenerService, devtronAppCMCSService DevtronAppCMCSService, ciPipelineConfigService CiPipelineConfigService, buildPipelineSwitchService BuildPipelineSwitchService, argoClientWrapperService argocdServer.ArgoClientWrapperService, - deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService) *CdPipelineConfigServiceImpl { + deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService, + gitOpsConfigReadService config.GitOpsConfigReadService, + gitOpsRemoteOperationService remote.GitOpsRemoteOperationService) *CdPipelineConfigServiceImpl { return &CdPipelineConfigServiceImpl{ logger: logger, pipelineRepository: pipelineRepository, @@ -184,11 +187,9 @@ func NewCdPipelineConfigServiceImpl(logger *zap.SugaredLogger, pipelineRepositor clusterRepository: clusterRepository, helmAppService: helmAppService, enforcerUtil: enforcerUtil, - gitOpsRepository: gitOpsRepository, pipelineStrategyHistoryService: pipelineStrategyHistoryService, chartRepository: chartRepository, resourceGroupService: resourceGroupService, - chartTemplateService: chartTemplateService, propertiesConfigService: propertiesConfigService, deploymentTemplateHistoryService: deploymentTemplateHistoryService, scopedVariableManager: scopedVariableManager, @@ -201,6 +202,8 @@ func NewCdPipelineConfigServiceImpl(logger *zap.SugaredLogger, pipelineRepositor buildPipelineSwitchService: buildPipelineSwitchService, argoClientWrapperService: argoClientWrapperService, deployedAppMetricsService: deployedAppMetricsService, + gitOpsConfigReadService: gitOpsConfigReadService, + gitOpsRemoteOperationService: gitOpsRemoteOperationService, } } @@ -341,8 +344,11 @@ func (impl *CdPipelineConfigServiceImpl) GetCdPipelineById(pipelineId int) (cdPi func (impl *CdPipelineConfigServiceImpl) CreateCdPipelines(pipelineCreateRequest *bean.CdPipelines, ctx context.Context) (*bean.CdPipelines, error) { //Validation for checking deployment App type - isGitOpsConfigured, err := impl.IsGitopsConfigured() - + isGitOpsConfigured, err := impl.gitOpsConfigReadService.IsGitOpsConfigured() + if err != nil { + impl.logger.Errorw("error in checking if gitOps is configured or not", "err", err) + return nil, err + } for _, pipeline := range pipelineCreateRequest.Pipelines { // skip creation of pipeline if envId is not set if pipeline.EnvironmentId <= 0 { @@ -1500,23 +1506,6 @@ func (impl *CdPipelineConfigServiceImpl) GetEnvironmentListForAutocompleteFilter return result, nil } -func (impl *CdPipelineConfigServiceImpl) IsGitopsConfigured() (bool, error) { - - isGitOpsConfigured := false - gitOpsConfig, err := impl.gitOpsRepository.GetGitOpsConfigActive() - - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("GetGitOpsConfigActive, error while getting", "err", err) - return false, err - } - if gitOpsConfig != nil && gitOpsConfig.Id > 0 { - isGitOpsConfigured = true - } - - return isGitOpsConfigured, nil - -} - func (impl *CdPipelineConfigServiceImpl) validateDeploymentAppType(pipeline *bean.CDPipelineConfigObject, deploymentConfig map[string]bool) error { // Config value doesn't exist in attribute table @@ -1600,15 +1589,14 @@ func (impl *CdPipelineConfigServiceImpl) ValidateCDPipelineRequest(pipelineCreat } -func (impl *CdPipelineConfigServiceImpl) RegisterInACD(gitOpsRepoName string, chartGitAttr *util.ChartGitAttribute, userId int32, ctx context.Context) error { - +func (impl *CdPipelineConfigServiceImpl) RegisterInACD(gitOpsRepoName string, chartGitAttr *commonBean.ChartGitAttribute, userId int32, ctx context.Context) error { err := impl.argoClientWrapperService.RegisterGitOpsRepoInArgo(ctx, chartGitAttr.RepoUrl) if err != nil { impl.logger.Errorw("error while register git repo in argo", "err", err) emptyRepoErrorMessage := []string{"failed to get index: 404 Not Found", "remote repository is empty"} if strings.Contains(err.Error(), emptyRepoErrorMessage[0]) || strings.Contains(err.Error(), emptyRepoErrorMessage[1]) { // - found empty repository, create some file in repository - err := impl.chartTemplateService.CreateReadmeInGitRepo(gitOpsRepoName, userId) + err := impl.gitOpsRemoteOperationService.CreateReadmeInGitRepo(gitOpsRepoName, userId) if err != nil { impl.logger.Errorw("error in creating file in git repo", "err", err) return err @@ -1920,7 +1908,7 @@ func (impl *CdPipelineConfigServiceImpl) updateCdPipeline(ctx context.Context, p return nil } -func (impl *CdPipelineConfigServiceImpl) updateGitRepoUrlInCharts(appId int, chartGitAttribute *util.ChartGitAttribute, userId int32) error { +func (impl *CdPipelineConfigServiceImpl) updateGitRepoUrlInCharts(appId int, chartGitAttribute *commonBean.ChartGitAttribute, userId int32) error { charts, err := impl.chartRepository.FindActiveChartsByAppId(appId) if err != nil && pg.ErrNoRows != err { return err diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index dedad255b4..1eebea4b9d 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -22,6 +22,7 @@ import ( "encoding/json" errors3 "errors" "fmt" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" "path" @@ -182,7 +183,6 @@ type WorkflowDagExecutorImpl struct { environmentConfigRepository chartConfig.EnvConfigOverrideRepository dbMigrationConfigRepository pipelineConfig.DbMigrationConfigRepository mergeUtil *util.MergeUtil - gitOpsConfigRepository repository.GitOpsConfigRepository gitFactory *util.GitFactory acdClient application2.ServiceClient argoClientWrapperService argocdServer.ArgoClientWrapperService @@ -190,6 +190,7 @@ type WorkflowDagExecutorImpl struct { ACDConfig *argocdServer.ACDConfig deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService chartRefService chartRef.ChartRefService + gitOpsConfigReadService config.GitOpsConfigReadService } const kedaAutoscaling = "kedaAutoscaling" @@ -284,7 +285,6 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi environmentConfigRepository chartConfig.EnvConfigOverrideRepository, dbMigrationConfigRepository pipelineConfig.DbMigrationConfigRepository, mergeUtil *util.MergeUtil, - gitOpsConfigRepository repository.GitOpsConfigRepository, gitFactory *util.GitFactory, acdClient application2.ServiceClient, argoClientWrapperService argocdServer.ArgoClientWrapperService, @@ -292,7 +292,8 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi customTagService CustomTagService, ACDConfig *argocdServer.ACDConfig, deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService, - chartRefService chartRef.ChartRefService) *WorkflowDagExecutorImpl { + chartRefService chartRef.ChartRefService, + gitOpsConfigReadService config.GitOpsConfigReadService) *WorkflowDagExecutorImpl { wde := &WorkflowDagExecutorImpl{logger: Logger, pipelineRepository: pipelineRepository, cdWorkflowRepository: cdWorkflowRepository, @@ -354,7 +355,6 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi environmentConfigRepository: environmentConfigRepository, dbMigrationConfigRepository: dbMigrationConfigRepository, mergeUtil: mergeUtil, - gitOpsConfigRepository: gitOpsConfigRepository, gitFactory: gitFactory, acdClient: acdClient, argoClientWrapperService: argoClientWrapperService, @@ -362,6 +362,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi ACDConfig: ACDConfig, deployedAppMetricsService: deployedAppMetricsService, chartRefService: chartRefService, + gitOpsConfigReadService: gitOpsConfigReadService, } config, err := types.GetCdConfig() if err != nil { @@ -4140,10 +4141,10 @@ func (impl *WorkflowDagExecutorImpl) mergeAndSave(envOverride *chartConfig.EnvCo commitHash := "" commitTime := time.Time{} if util.IsAcdApp(pipeline.DeploymentAppType) { - chartRepoName := impl.chartTemplateService.GetGitOpsRepoNameFromUrl(envOverride.Chart.GitRepoUrl) + chartRepoName := impl.gitOpsConfigReadService.GetGitOpsRepoNameFromUrl(envOverride.Chart.GitRepoUrl) _, span = otel.Tracer("orchestrator").Start(ctx, "chartTemplateService.GetUserEmailIdAndNameForGitOpsCommit") //getting username & emailId for commit author data - userEmailId, userName := impl.chartTemplateService.GetUserEmailIdAndNameForGitOpsCommit(overrideRequest.UserId) + userEmailId, userName := impl.gitOpsConfigReadService.GetUserEmailIdAndNameForGitOpsCommit(overrideRequest.UserId) span.End() chartGitAttr := &util.ChartConfig{ FileName: fmt.Sprintf("_%d-values.yaml", envOverride.TargetEnvironment), @@ -4155,15 +4156,12 @@ func (impl *WorkflowDagExecutorImpl) mergeAndSave(envOverride *chartConfig.EnvCo UserName: userName, UserEmailId: userEmailId, } - gitOpsConfigBitbucket, err := impl.gitOpsConfigRepository.GetGitOpsConfigByProvider(util.BITBUCKET_PROVIDER) + bitbucketMetadata, err := impl.gitOpsConfigReadService.GetBitbucketMetadata() if err != nil { - if err == pg.ErrNoRows { - gitOpsConfigBitbucket.BitBucketWorkspaceId = "" - } else { - return 0, 0, "", err - } + impl.logger.Errorw("error in getting bitbucket metadata", "err", err) + return 0, 0, "", err } - gitOpsConfig := &bean.GitOpsConfigDto{BitBucketWorkspaceId: gitOpsConfigBitbucket.BitBucketWorkspaceId} + gitOpsConfig := &bean.GitOpsConfigDto{BitBucketWorkspaceId: bitbucketMetadata.BitBucketWorkspaceId} _, span = otel.Tracer("orchestrator").Start(ctx, "gitFactory.Client.CommitValues") commitHash, commitTime, err = impl.gitFactory.Client.CommitValues(chartGitAttr, gitOpsConfig) span.End() diff --git a/wire_gen.go b/wire_gen.go index b010214e4a..420e078477 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -101,6 +101,8 @@ import ( "github.com/devtron-labs/devtron/pkg/clusterTerminalAccess" "github.com/devtron-labs/devtron/pkg/commonService" delete2 "github.com/devtron-labs/devtron/pkg/delete" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" repository11 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/repository" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate" @@ -167,11 +169,11 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - config, err := sql.GetConfig() + sqlConfig, err := sql.GetConfig() if err != nil { return nil, err } - db, err := sql.NewDbConnection(config, sugaredLogger) + db, err := sql.NewDbConnection(sqlConfig, sugaredLogger) if err != nil { return nil, err } @@ -299,17 +301,8 @@ func InitializeApp() (*App, error) { chartRepositoryImpl := chartRepoRepository.NewChartRepository(db) dockerArtifactStoreRepositoryImpl := repository5.NewDockerArtifactStoreRepositoryImpl(db) gitProviderRepositoryImpl := repository.NewGitProviderRepositoryImpl(db) - commonServiceImpl := commonService.NewCommonServiceImpl(sugaredLogger, chartRepositoryImpl, envConfigOverrideRepositoryImpl, gitOpsConfigRepositoryImpl, dockerArtifactStoreRepositoryImpl, attributesRepositoryImpl, gitProviderRepositoryImpl, environmentRepositoryImpl, teamRepositoryImpl, appRepositoryImpl) - gitCliUtil := util.NewGitCliUtil(sugaredLogger) - gitFactory, err := util.NewGitFactory(sugaredLogger, gitOpsConfigRepositoryImpl, gitCliUtil) - if err != nil { - return nil, err - } - globalEnvVariables, err := util2.GetGlobalEnvVariables() - if err != nil { - return nil, err - } - chartTemplateServiceImpl := util.NewChartTemplateServiceImpl(sugaredLogger, gitFactory, globalEnvVariables, gitOpsConfigRepositoryImpl, userRepositoryImpl, chartRepositoryImpl) + commonServiceImpl := commonService.NewCommonServiceImpl(sugaredLogger, chartRepositoryImpl, envConfigOverrideRepositoryImpl, dockerArtifactStoreRepositoryImpl, attributesRepositoryImpl, gitProviderRepositoryImpl, environmentRepositoryImpl, teamRepositoryImpl, appRepositoryImpl) + chartTemplateServiceImpl := util.NewChartTemplateServiceImpl(sugaredLogger) devtronSecretConfig, err := util2.GetDevtronSecretName() if err != nil { return nil, err @@ -334,6 +327,10 @@ func InitializeApp() (*App, error) { enforcerImpl := casbin.NewEnforcerImpl(syncedEnforcer, sessionManager, sugaredLogger) enforcerUtilImpl := rbac.NewEnforcerUtilImpl(sugaredLogger, teamRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, clusterRepositoryImpl, enforcerImpl) appStatusServiceImpl := appStatus2.NewAppStatusServiceImpl(appStatusRepositoryImpl, sugaredLogger, enforcerImpl, enforcerUtilImpl) + globalEnvVariables, err := util2.GetGlobalEnvVariables() + if err != nil { + return nil, err + } scopedVariableRepositoryImpl := repository6.NewScopedVariableRepository(db, sugaredLogger) qualifiersMappingRepositoryImpl, err := resourceQualifiers.NewQualifiersMappingRepositoryImpl(db, sugaredLogger) if err != nil { @@ -373,7 +370,14 @@ func InitializeApp() (*App, error) { Logger: sugaredLogger, } chartRefServiceImpl := chartRef.NewChartRefServiceImpl(sugaredLogger, chartRefRepositoryImpl, chartTemplateServiceImpl, utilMergeUtil) - appServiceImpl := app2.NewAppService(envConfigOverrideRepositoryImpl, pipelineOverrideRepositoryImpl, mergeUtil, sugaredLogger, pipelineRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, applicationServiceClientImpl, appRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, chartTemplateServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, appStatusServiceImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, globalEnvVariables, scopedVariableCMCSManagerImpl, acdConfig, chartRefServiceImpl) + gitOpsConfigReadServiceImpl := config.NewGitOpsConfigReadServiceImpl(sugaredLogger, gitOpsConfigRepositoryImpl, userServiceImpl, globalEnvVariables) + gitCliUtil := util.NewGitCliUtil(sugaredLogger) + gitFactory, err := util.NewGitFactory(sugaredLogger, gitOpsConfigRepositoryImpl, gitCliUtil) + if err != nil { + return nil, err + } + gitOpsRemoteOperationServiceImpl := remote.NewGitOpsRemoteOperationServiceImpl(sugaredLogger, gitFactory, gitOpsConfigReadServiceImpl, chartTemplateServiceImpl) + appServiceImpl := app2.NewAppService(envConfigOverrideRepositoryImpl, pipelineOverrideRepositoryImpl, mergeUtil, sugaredLogger, pipelineRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, applicationServiceClientImpl, appRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, chartTemplateServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, appStatusServiceImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, globalEnvVariables, scopedVariableCMCSManagerImpl, acdConfig, chartRefServiceImpl, gitOpsConfigReadServiceImpl, gitOpsRemoteOperationServiceImpl) validate, err := util.IntValidator() if err != nil { return nil, err @@ -432,7 +436,7 @@ func InitializeApp() (*App, error) { pipelineStrategyHistoryRepositoryImpl := repository8.NewPipelineStrategyHistoryRepositoryImpl(sugaredLogger, db) pipelineStrategyHistoryServiceImpl := history.NewPipelineStrategyHistoryServiceImpl(sugaredLogger, pipelineStrategyHistoryRepositoryImpl, userServiceImpl) manifestPushConfigRepositoryImpl := repository9.NewManifestPushConfigRepository(sugaredLogger, db) - gitOpsManifestPushServiceImpl := app2.NewGitOpsManifestPushServiceImpl(sugaredLogger, chartTemplateServiceImpl, gitOpsConfigRepositoryImpl, gitFactory, pipelineStatusTimelineServiceImpl, pipelineStatusTimelineRepositoryImpl, acdConfig, chartRefServiceImpl) + gitOpsManifestPushServiceImpl := app2.NewGitOpsManifestPushServiceImpl(sugaredLogger, gitFactory, pipelineStatusTimelineServiceImpl, pipelineStatusTimelineRepositoryImpl, acdConfig, chartRefServiceImpl, gitOpsConfigReadServiceImpl, gitOpsRemoteOperationServiceImpl) imageScanHistoryRepositoryImpl := security.NewImageScanHistoryRepositoryImpl(db, sugaredLogger) imageScanDeployInfoRepositoryImpl := security.NewImageScanDeployInfoRepositoryImpl(db, sugaredLogger) genericNoteRepositoryImpl := repository12.NewGenericNoteRepositoryImpl(db) @@ -448,7 +452,7 @@ func InitializeApp() (*App, error) { repositoryServiceClientImpl := repository13.NewServiceClientImpl(sugaredLogger, argoCDConnectionManagerImpl) argoClientWrapperServiceImpl := argocdServer.NewArgoClientWrapperServiceImpl(sugaredLogger, applicationServiceClientImpl, acdConfig, repositoryServiceClientImpl) pipelineConfigListenerServiceImpl := pipeline.NewPipelineConfigListenerServiceImpl(sugaredLogger) - workflowDagExecutorImpl := pipeline.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, pubSubClientServiceImpl, appServiceImpl, workflowServiceImpl, ciArtifactRepositoryImpl, ciPipelineRepositoryImpl, materialRepositoryImpl, pipelineOverrideRepositoryImpl, userServiceImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, enforcerUtilImpl, eventSimpleFactoryImpl, eventRESTClientImpl, cvePolicyRepositoryImpl, imageScanResultRepositoryImpl, appWorkflowRepositoryImpl, prePostCdScriptHistoryServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineServiceImpl, ciTemplateRepositoryImpl, ciWorkflowRepositoryImpl, appLabelRepositoryImpl, clientImpl, pipelineStageServiceImpl, k8sCommonServiceImpl, globalPluginServiceImpl, pluginInputVariableParserImpl, scopedVariableCMCSManagerImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineStrategyHistoryServiceImpl, manifestPushConfigRepositoryImpl, gitOpsManifestPushServiceImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, appCrudOperationServiceImpl, pipelineConfigRepositoryImpl, dockerRegistryIpsConfigServiceImpl, chartRepositoryImpl, chartTemplateServiceImpl, pipelineStrategyHistoryRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, argoK8sClientImpl, configMapRepositoryImpl, configMapHistoryRepositoryImpl, helmAppServiceImpl, helmAppClientImpl, envConfigOverrideRepositoryImpl, dbMigrationConfigRepositoryImpl, mergeUtil, gitOpsConfigRepositoryImpl, gitFactory, applicationServiceClientImpl, argoClientWrapperServiceImpl, pipelineConfigListenerServiceImpl, customTagServiceImpl, acdConfig, deployedAppMetricsServiceImpl, chartRefServiceImpl) + workflowDagExecutorImpl := pipeline.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, pubSubClientServiceImpl, appServiceImpl, workflowServiceImpl, ciArtifactRepositoryImpl, ciPipelineRepositoryImpl, materialRepositoryImpl, pipelineOverrideRepositoryImpl, userServiceImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, enforcerUtilImpl, eventSimpleFactoryImpl, eventRESTClientImpl, cvePolicyRepositoryImpl, imageScanResultRepositoryImpl, appWorkflowRepositoryImpl, prePostCdScriptHistoryServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineServiceImpl, ciTemplateRepositoryImpl, ciWorkflowRepositoryImpl, appLabelRepositoryImpl, clientImpl, pipelineStageServiceImpl, k8sCommonServiceImpl, globalPluginServiceImpl, pluginInputVariableParserImpl, scopedVariableCMCSManagerImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineStrategyHistoryServiceImpl, manifestPushConfigRepositoryImpl, gitOpsManifestPushServiceImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, appCrudOperationServiceImpl, pipelineConfigRepositoryImpl, dockerRegistryIpsConfigServiceImpl, chartRepositoryImpl, chartTemplateServiceImpl, pipelineStrategyHistoryRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, argoK8sClientImpl, configMapRepositoryImpl, configMapHistoryRepositoryImpl, helmAppServiceImpl, helmAppClientImpl, envConfigOverrideRepositoryImpl, dbMigrationConfigRepositoryImpl, mergeUtil, gitFactory, applicationServiceClientImpl, argoClientWrapperServiceImpl, pipelineConfigListenerServiceImpl, customTagServiceImpl, acdConfig, deployedAppMetricsServiceImpl, chartRefServiceImpl, gitOpsConfigReadServiceImpl) deploymentGroupAppRepositoryImpl := repository.NewDeploymentGroupAppRepositoryImpl(sugaredLogger, db) deploymentGroupServiceImpl := deploymentGroup.NewDeploymentGroupServiceImpl(appRepositoryImpl, sugaredLogger, pipelineRepositoryImpl, ciPipelineRepositoryImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, deploymentGroupAppRepositoryImpl, ciArtifactRepositoryImpl, appWorkflowRepositoryImpl, workflowDagExecutorImpl) deploymentConfigServiceImpl := pipeline.NewDeploymentConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, pipelineRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, configMapHistoryServiceImpl, scopedVariableCMCSManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) @@ -489,15 +493,15 @@ func InitializeApp() (*App, error) { return nil, err } devtronAppCMCSServiceImpl := pipeline.NewDevtronAppCMCSServiceImpl(sugaredLogger, appServiceImpl, attributesRepositoryImpl) - cdPipelineConfigServiceImpl := pipeline.NewCdPipelineConfigServiceImpl(sugaredLogger, pipelineRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, appRepositoryImpl, appServiceImpl, deploymentGroupRepositoryImpl, ciCdPipelineOrchestratorImpl, appStatusRepositoryImpl, ciPipelineRepositoryImpl, prePostCdScriptHistoryServiceImpl, clusterRepositoryImpl, helmAppServiceImpl, enforcerUtilImpl, gitOpsConfigRepositoryImpl, pipelineStrategyHistoryServiceImpl, chartRepositoryImpl, resourceGroupServiceImpl, chartTemplateServiceImpl, propertiesConfigServiceImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deploymentServiceTypeConfig, applicationServiceClientImpl, customTagServiceImpl, pipelineConfigListenerServiceImpl, devtronAppCMCSServiceImpl, ciPipelineConfigServiceImpl, buildPipelineSwitchServiceImpl, argoClientWrapperServiceImpl, deployedAppMetricsServiceImpl) + cdPipelineConfigServiceImpl := pipeline.NewCdPipelineConfigServiceImpl(sugaredLogger, pipelineRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, appRepositoryImpl, appServiceImpl, deploymentGroupRepositoryImpl, ciCdPipelineOrchestratorImpl, appStatusRepositoryImpl, ciPipelineRepositoryImpl, prePostCdScriptHistoryServiceImpl, clusterRepositoryImpl, helmAppServiceImpl, enforcerUtilImpl, pipelineStrategyHistoryServiceImpl, chartRepositoryImpl, resourceGroupServiceImpl, propertiesConfigServiceImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deploymentServiceTypeConfig, applicationServiceClientImpl, customTagServiceImpl, pipelineConfigListenerServiceImpl, devtronAppCMCSServiceImpl, ciPipelineConfigServiceImpl, buildPipelineSwitchServiceImpl, argoClientWrapperServiceImpl, deployedAppMetricsServiceImpl, gitOpsConfigReadServiceImpl, gitOpsRemoteOperationServiceImpl) appArtifactManagerImpl := pipeline.NewAppArtifactManagerImpl(sugaredLogger, cdWorkflowRepositoryImpl, userServiceImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, ciWorkflowRepositoryImpl, pipelineStageServiceImpl, cdPipelineConfigServiceImpl, dockerArtifactStoreRepositoryImpl, ciPipelineRepositoryImpl, ciTemplateServiceImpl) globalStrategyMetadataChartRefMappingRepositoryImpl := chartRepoRepository.NewGlobalStrategyMetadataChartRefMappingRepositoryImpl(db, sugaredLogger) devtronAppStrategyServiceImpl := pipeline.NewDevtronAppStrategyServiceImpl(sugaredLogger, chartRepositoryImpl, globalStrategyMetadataChartRefMappingRepositoryImpl, ciCdPipelineOrchestratorImpl, cdPipelineConfigServiceImpl) - appDeploymentTypeChangeManagerImpl := pipeline.NewAppDeploymentTypeChangeManagerImpl(sugaredLogger, pipelineRepositoryImpl, workflowDagExecutorImpl, appServiceImpl, chartTemplateServiceImpl, appStatusRepositoryImpl, helmAppServiceImpl, applicationServiceClientImpl, appArtifactManagerImpl, cdPipelineConfigServiceImpl) + chartServiceImpl := chart.NewChartServiceImpl(chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, appRepositoryImpl, utilMergeUtil, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, environmentRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) + appDeploymentTypeChangeManagerImpl := pipeline.NewAppDeploymentTypeChangeManagerImpl(sugaredLogger, pipelineRepositoryImpl, workflowDagExecutorImpl, appServiceImpl, appStatusRepositoryImpl, helmAppServiceImpl, applicationServiceClientImpl, appArtifactManagerImpl, cdPipelineConfigServiceImpl, gitOpsConfigReadServiceImpl, chartServiceImpl) devtronAppConfigServiceImpl := pipeline.NewDevtronAppConfigServiceImpl(sugaredLogger, ciCdPipelineOrchestratorImpl, appRepositoryImpl, pipelineRepositoryImpl, resourceGroupServiceImpl, enforcerUtilImpl, ciMaterialConfigServiceImpl) pipelineBuilderImpl := pipeline.NewPipelineBuilderImpl(sugaredLogger, materialRepositoryImpl, chartRepositoryImpl, ciPipelineConfigServiceImpl, ciMaterialConfigServiceImpl, appArtifactManagerImpl, devtronAppCMCSServiceImpl, devtronAppStrategyServiceImpl, appDeploymentTypeChangeManagerImpl, cdPipelineConfigServiceImpl, devtronAppConfigServiceImpl) deploymentTemplateValidationServiceImpl := deploymentTemplate.NewDeploymentTemplateValidationServiceImpl(sugaredLogger, chartRefServiceImpl, scopedVariableManagerImpl) - chartServiceImpl := chart.NewChartServiceImpl(chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, appRepositoryImpl, utilMergeUtil, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, environmentRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) dbMigrationServiceImpl := pipeline.NewDbMogrationService(sugaredLogger, dbMigrationConfigRepositoryImpl) ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciWorkflowRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, mergeUtil, ciPipelineRepositoryImpl, prePostCiScriptHistoryServiceImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateServiceImpl, appCrudOperationServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, customTagServiceImpl, pluginInputVariableParserImpl, globalPluginServiceImpl) ciLogServiceImpl, err := pipeline.NewCiLogServiceImpl(sugaredLogger, ciServiceImpl, k8sUtil) @@ -513,7 +517,7 @@ func InitializeApp() (*App, error) { deploymentEventHandlerImpl := app2.NewDeploymentEventHandlerImpl(sugaredLogger, appListingServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl) cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, helmAppServiceImpl, pipelineOverrideRepositoryImpl, workflowDagExecutorImpl, appListingServiceImpl, appListingRepositoryImpl, pipelineStatusTimelineRepositoryImpl, applicationServiceClientImpl, argoUserServiceImpl, deploymentEventHandlerImpl, eventRESTClientImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceImpl, appStatusServiceImpl, enforcerUtilImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, appRepositoryImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sUtil, workflowServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl, customTagServiceImpl, argoClientWrapperServiceImpl, appServiceConfig, acdConfig) appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, appRepositoryImpl, userAuthServiceImpl) - appCloneServiceImpl := appClone.NewAppCloneServiceImpl(sugaredLogger, pipelineBuilderImpl, materialRepositoryImpl, chartServiceImpl, configMapServiceImpl, appWorkflowServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, ciTemplateOverrideRepositoryImpl, pipelineStageServiceImpl, ciTemplateServiceImpl, appRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, appWorkflowRepositoryImpl, ciPipelineConfigServiceImpl) + appCloneServiceImpl := appClone.NewAppCloneServiceImpl(sugaredLogger, pipelineBuilderImpl, chartServiceImpl, configMapServiceImpl, appWorkflowServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, pipelineStageServiceImpl, ciTemplateServiceImpl, appRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, ciPipelineConfigServiceImpl, gitOpsConfigReadServiceImpl) deploymentTemplateRepositoryImpl := repository.NewDeploymentTemplateRepositoryImpl(db, sugaredLogger) deploymentTemplateServiceImpl := generateManifest.NewDeploymentTemplateServiceImpl(sugaredLogger, chartServiceImpl, appListingServiceImpl, appListingRepositoryImpl, deploymentTemplateRepositoryImpl, helmAppServiceImpl, chartRepositoryImpl, chartTemplateServiceImpl, helmAppClientImpl, k8sUtil, propertiesConfigServiceImpl, deploymentTemplateHistoryServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, chartRefServiceImpl) imageScanObjectMetaRepositoryImpl := security.NewImageScanObjectMetaRepositoryImpl(db, sugaredLogger) @@ -540,16 +544,16 @@ func InitializeApp() (*App, error) { } tokenCache := util3.NewTokenCache(sugaredLogger, acdAuthConfig, userAuthServiceImpl) chartGroupDeploymentRepositoryImpl := repository3.NewChartGroupDeploymentRepositoryImpl(db, sugaredLogger) - appStoreDeploymentCommonServiceImpl := appStoreDeploymentCommon.NewAppStoreDeploymentCommonServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, chartTemplateServiceImpl, gitFactory, gitOpsConfigRepositoryImpl) - appStoreDeploymentFullModeServiceImpl := appStoreDeploymentFullMode.NewAppStoreDeploymentFullModeServiceImpl(sugaredLogger, chartTemplateServiceImpl, repositoryServiceClientImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, applicationServiceClientImpl, argoK8sClientImpl, gitFactory, acdAuthConfig, globalEnvVariables, installedAppRepositoryImpl, tokenCache, argoUserServiceImpl, gitOpsConfigRepositoryImpl, pipelineStatusTimelineServiceImpl, appStoreDeploymentCommonServiceImpl, argoClientWrapperServiceImpl, pubSubClientServiceImpl, installedAppVersionHistoryRepositoryImpl, acdConfig) + appStoreDeploymentCommonServiceImpl := appStoreDeploymentCommon.NewAppStoreDeploymentCommonServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, chartTemplateServiceImpl, gitFactory, gitOpsConfigReadServiceImpl, gitOpsRemoteOperationServiceImpl) + appStoreDeploymentFullModeServiceImpl := appStoreDeploymentFullMode.NewAppStoreDeploymentFullModeServiceImpl(sugaredLogger, applicationServiceClientImpl, argoK8sClientImpl, acdAuthConfig, argoUserServiceImpl, pipelineStatusTimelineServiceImpl, appStoreDeploymentCommonServiceImpl, argoClientWrapperServiceImpl, pubSubClientServiceImpl, installedAppVersionHistoryRepositoryImpl, acdConfig, gitOpsConfigReadServiceImpl) clusterInstalledAppsRepositoryImpl := repository3.NewClusterInstalledAppsRepositoryImpl(db, sugaredLogger) appStoreDeploymentHelmServiceImpl := appStoreDeploymentTool.NewAppStoreDeploymentHelmServiceImpl(sugaredLogger, helmAppServiceImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, helmAppClientImpl, installedAppRepositoryImpl, appStoreDeploymentCommonServiceImpl, ociRegistryConfigRepositoryImpl) - appStoreDeploymentArgoCdServiceImpl := appStoreDeploymentGitopsTool.NewAppStoreDeploymentArgoCdServiceImpl(sugaredLogger, appStoreDeploymentFullModeServiceImpl, applicationServiceClientImpl, chartGroupDeploymentRepositoryImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, chartTemplateServiceImpl, gitFactory, argoUserServiceImpl, appStoreDeploymentCommonServiceImpl, helmAppServiceImpl, gitOpsConfigRepositoryImpl, appStatusServiceImpl, pipelineStatusTimelineServiceImpl, userServiceImpl, pipelineStatusTimelineRepositoryImpl, appStoreApplicationVersionRepositoryImpl, argoClientWrapperServiceImpl, acdConfig) + appStoreDeploymentArgoCdServiceImpl := appStoreDeploymentGitopsTool.NewAppStoreDeploymentArgoCdServiceImpl(sugaredLogger, appStoreDeploymentFullModeServiceImpl, applicationServiceClientImpl, chartGroupDeploymentRepositoryImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, gitFactory, argoUserServiceImpl, appStoreDeploymentCommonServiceImpl, helmAppServiceImpl, appStatusServiceImpl, pipelineStatusTimelineServiceImpl, userServiceImpl, pipelineStatusTimelineRepositoryImpl, appStoreApplicationVersionRepositoryImpl, argoClientWrapperServiceImpl, acdConfig) serviceDeploymentServiceTypeConfig, err := service2.GetDeploymentServiceTypeConfig() if err != nil { return nil, err } - appStoreDeploymentServiceImpl := service2.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, clusterInstalledAppsRepositoryImpl, appRepositoryImpl, appStoreDeploymentHelmServiceImpl, appStoreDeploymentArgoCdServiceImpl, environmentServiceImpl, clusterServiceImplExtended, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, globalEnvVariables, installedAppVersionHistoryRepositoryImpl, gitOpsConfigRepositoryImpl, attributesServiceImpl, serviceDeploymentServiceTypeConfig, chartTemplateServiceImpl, acdConfig) + appStoreDeploymentServiceImpl := service2.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, clusterInstalledAppsRepositoryImpl, appRepositoryImpl, appStoreDeploymentHelmServiceImpl, appStoreDeploymentArgoCdServiceImpl, environmentServiceImpl, clusterServiceImplExtended, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, globalEnvVariables, installedAppVersionHistoryRepositoryImpl, gitOpsConfigRepositoryImpl, serviceDeploymentServiceTypeConfig, acdConfig, gitOpsConfigReadServiceImpl) k8sResourceHistoryRepositoryImpl := repository15.NewK8sResourceHistoryRepositoryImpl(db, sugaredLogger) k8sResourceHistoryServiceImpl := kubernetesResourceAuditLogs.Newk8sResourceHistoryServiceImpl(k8sResourceHistoryRepositoryImpl, sugaredLogger, appRepositoryImpl, environmentRepositoryImpl) ephemeralContainersRepositoryImpl := repository2.NewEphemeralContainersRepositoryImpl(db) @@ -559,7 +563,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - installedAppServiceImpl, err := service2.NewInstalledAppServiceImpl(sugaredLogger, installedAppRepositoryImpl, chartTemplateServiceImpl, repositoryServiceClientImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, teamRepositoryImpl, appRepositoryImpl, applicationServiceClientImpl, appStoreValuesServiceImpl, pubSubClientServiceImpl, tokenCache, chartGroupDeploymentRepositoryImpl, environmentServiceImpl, argoK8sClientImpl, gitFactory, acdAuthConfig, gitOpsConfigRepositoryImpl, userServiceImpl, appStoreDeploymentFullModeServiceImpl, appStoreDeploymentServiceImpl, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, helmAppClientImpl, helmAppServiceImpl, appStatusServiceImpl, k8sUtil, pipelineStatusTimelineServiceImpl, appStoreDeploymentCommonServiceImpl, k8sCommonServiceImpl, k8sApplicationServiceImpl, acdConfig) + installedAppServiceImpl, err := service2.NewInstalledAppServiceImpl(sugaredLogger, installedAppRepositoryImpl, repositoryServiceClientImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, teamRepositoryImpl, appRepositoryImpl, applicationServiceClientImpl, appStoreValuesServiceImpl, pubSubClientServiceImpl, tokenCache, chartGroupDeploymentRepositoryImpl, environmentServiceImpl, argoK8sClientImpl, gitFactory, acdAuthConfig, gitOpsConfigRepositoryImpl, userServiceImpl, appStoreDeploymentFullModeServiceImpl, appStoreDeploymentServiceImpl, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, helmAppClientImpl, helmAppServiceImpl, appStatusServiceImpl, k8sUtil, pipelineStatusTimelineServiceImpl, appStoreDeploymentCommonServiceImpl, k8sCommonServiceImpl, k8sApplicationServiceImpl, acdConfig, gitOpsConfigReadServiceImpl) if err != nil { return nil, err } @@ -697,8 +701,8 @@ func InitializeApp() (*App, error) { imageScanRouterImpl := router.NewImageScanRouterImpl(imageScanRestHandlerImpl) policyRestHandlerImpl := restHandler.NewPolicyRestHandlerImpl(sugaredLogger, policyServiceImpl, userServiceImpl, userAuthServiceImpl, enforcerImpl, enforcerUtilImpl, environmentServiceImpl) policyRouterImpl := router.NewPolicyRouterImpl(policyRestHandlerImpl) - gitOpsConfigServiceImpl := gitops.NewGitOpsConfigServiceImpl(sugaredLogger, globalEnvVariables, gitOpsConfigRepositoryImpl, k8sUtil, acdAuthConfig, clusterServiceImplExtended, environmentServiceImpl, versionServiceImpl, gitFactory, chartTemplateServiceImpl, argoUserServiceImpl, serviceClientImpl) - gitOpsConfigRestHandlerImpl := restHandler.NewGitOpsConfigRestHandlerImpl(sugaredLogger, gitOpsConfigServiceImpl, userServiceImpl, validate, enforcerImpl, teamServiceImpl, gitOpsConfigRepositoryImpl) + gitOpsConfigServiceImpl := gitops.NewGitOpsConfigServiceImpl(sugaredLogger, globalEnvVariables, gitOpsConfigRepositoryImpl, k8sUtil, acdAuthConfig, clusterServiceImplExtended, gitFactory, argoUserServiceImpl, serviceClientImpl, gitOpsConfigReadServiceImpl) + gitOpsConfigRestHandlerImpl := restHandler.NewGitOpsConfigRestHandlerImpl(sugaredLogger, gitOpsConfigServiceImpl, userServiceImpl, validate, enforcerImpl, teamServiceImpl) gitOpsConfigRouterImpl := router.NewGitOpsConfigRouterImpl(gitOpsConfigRestHandlerImpl) dashboardConfig, err := dashboard.GetConfig() if err != nil { @@ -711,7 +715,7 @@ func InitializeApp() (*App, error) { userAttributesServiceImpl := attributes.NewUserAttributesServiceImpl(sugaredLogger, userAttributesRepositoryImpl) userAttributesRestHandlerImpl := restHandler.NewUserAttributesRestHandlerImpl(sugaredLogger, enforcerImpl, userServiceImpl, userAttributesServiceImpl) userAttributesRouterImpl := router.NewUserAttributesRouterImpl(userAttributesRestHandlerImpl) - commonRestHanlderImpl := restHandler.NewCommonRestHanlderImpl(sugaredLogger, gitOpsConfigServiceImpl, userServiceImpl, validate, enforcerImpl, commonServiceImpl) + commonRestHanlderImpl := restHandler.NewCommonRestHanlderImpl(sugaredLogger, userServiceImpl, commonServiceImpl) commonRouterImpl := router.NewCommonRouterImpl(commonRestHanlderImpl) grafanaConfig, err := grafana.GetConfig() if err != nil { From 347e0849149f2fdb5b5ea6ffd3c0079bbce552b3 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Thu, 18 Jan 2024 13:34:53 +0530 Subject: [PATCH 24/64] moved gitClient code to a common wrapper service --- cmd/external-app/wire_gen.go | 4 +- internal/util/GitService.go | 1 - pkg/app/ManifestPushService.go | 26 ++------ .../common/AppStoreDeploymentCommonService.go | 35 ++--------- .../AppStoreDeploymentFullModeService.go | 10 +++- .../service/AppStoreDeploymentService.go | 12 ++-- .../deployment/service/InstalledAppService.go | 27 ++++----- .../tool/AppStoreDeploymentHelmService.go | 15 +++-- .../gitops/AppStoreDeploymentArgoCdService.go | 13 ++-- .../remote/GitOpsRemoteOperationService.go | 59 +++++++++++++++++++ pkg/pipeline/WorkflowDagExecutor.go | 24 +++----- wire_gen.go | 14 ++--- 12 files changed, 129 insertions(+), 111 deletions(-) diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index f664986f3c..da0adc1866 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -274,7 +274,7 @@ func InitializeApp() (*App, error) { appStoreValuesRouterImpl := appStoreValues.NewAppStoreValuesRouterImpl(appStoreValuesRestHandlerImpl) chartGroupDeploymentRepositoryImpl := repository4.NewChartGroupDeploymentRepositoryImpl(db, sugaredLogger) clusterInstalledAppsRepositoryImpl := repository4.NewClusterInstalledAppsRepositoryImpl(db, sugaredLogger) - appStoreDeploymentHelmServiceImpl := appStoreDeploymentTool.NewAppStoreDeploymentHelmServiceImpl(sugaredLogger, helmAppServiceImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, helmAppClientImpl, installedAppRepositoryImpl, appStoreDeploymentCommonServiceImpl, ociRegistryConfigRepositoryImpl) + appStoreDeploymentHelmServiceImpl := appStoreDeploymentTool.NewAppStoreDeploymentHelmServiceImpl(sugaredLogger, helmAppServiceImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, helmAppClientImpl, installedAppRepositoryImpl, appStoreDeploymentCommonServiceImpl, ociRegistryConfigRepositoryImpl, gitOpsRemoteOperationServiceImpl) installedAppVersionHistoryRepositoryImpl := repository4.NewInstalledAppVersionHistoryRepositoryImpl(sugaredLogger, db) deploymentServiceTypeConfig, err := service3.GetDeploymentServiceTypeConfig() if err != nil { @@ -284,7 +284,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - appStoreDeploymentServiceImpl := service3.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, clusterInstalledAppsRepositoryImpl, appRepositoryImpl, appStoreDeploymentHelmServiceImpl, appStoreDeploymentHelmServiceImpl, environmentServiceImpl, clusterServiceImpl, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, globalEnvVariables, installedAppVersionHistoryRepositoryImpl, gitOpsConfigRepositoryImpl, deploymentServiceTypeConfig, acdConfig, gitOpsConfigReadServiceImpl) + appStoreDeploymentServiceImpl := service3.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, clusterInstalledAppsRepositoryImpl, appRepositoryImpl, appStoreDeploymentHelmServiceImpl, appStoreDeploymentHelmServiceImpl, environmentServiceImpl, clusterServiceImpl, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, globalEnvVariables, installedAppVersionHistoryRepositoryImpl, gitOpsConfigRepositoryImpl, deploymentServiceTypeConfig, acdConfig, gitOpsConfigReadServiceImpl, gitOpsRemoteOperationServiceImpl) appStoreDeploymentRestHandlerImpl := appStoreDeployment.NewAppStoreDeploymentRestHandlerImpl(sugaredLogger, userServiceImpl, enforcerImpl, enforcerUtilImpl, enforcerUtilHelmImpl, appStoreDeploymentServiceImpl, validate, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, helmUserServiceImpl, attributesServiceImpl) appStoreDeploymentRouterImpl := appStoreDeployment.NewAppStoreDeploymentRouterImpl(appStoreDeploymentRestHandlerImpl) chartProviderServiceImpl := chartProvider.NewChartProviderServiceImpl(sugaredLogger, chartRepoRepositoryImpl, chartRepositoryServiceImpl, dockerArtifactStoreRepositoryImpl, ociRegistryConfigRepositoryImpl) diff --git a/internal/util/GitService.go b/internal/util/GitService.go index 06b2a47d63..882772e523 100644 --- a/internal/util/GitService.go +++ b/internal/util/GitService.go @@ -52,7 +52,6 @@ const ( ) type GitClient interface { - //TODO: make all methods independent of GitOpsConfigDto, create own dto object CreateRepository(config *bean2.GitOpsConfigDto) (url string, isNew bool, detailedErrorGitOpsConfigActions DetailedErrorGitOpsConfigActions) CommitValues(config *ChartConfig, gitOpsConfig *bean2.GitOpsConfigDto) (commitHash string, commitTime time.Time, err error) GetRepoUrl(config *bean2.GitOpsConfigDto) (repoUrl string, err error) diff --git a/pkg/app/ManifestPushService.go b/pkg/app/ManifestPushService.go index 1e32701e53..937023f39b 100644 --- a/pkg/app/ManifestPushService.go +++ b/pkg/app/ManifestPushService.go @@ -3,11 +3,9 @@ package app import ( "context" "fmt" - bean2 "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/client/argocdServer" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" - . "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app/bean" status2 "github.com/devtron-labs/devtron/pkg/app/status" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" @@ -28,7 +26,6 @@ type GitOpsPushService interface { type GitOpsManifestPushServiceImpl struct { logger *zap.SugaredLogger - gitFactory *GitFactory pipelineStatusTimelineService status2.PipelineStatusTimelineService pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository acdConfig *argocdServer.ACDConfig @@ -37,9 +34,7 @@ type GitOpsManifestPushServiceImpl struct { gitOpsRemoteOperationService remote.GitOpsRemoteOperationService } -func NewGitOpsManifestPushServiceImpl( - logger *zap.SugaredLogger, - gitFactory *GitFactory, +func NewGitOpsManifestPushServiceImpl(logger *zap.SugaredLogger, pipelineStatusTimelineService status2.PipelineStatusTimelineService, pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository, acdConfig *argocdServer.ACDConfig, chartRefService chartRef.ChartRefService, @@ -47,7 +42,6 @@ func NewGitOpsManifestPushServiceImpl( gitOpsRemoteOperationService remote.GitOpsRemoteOperationService) *GitOpsManifestPushServiceImpl { return &GitOpsManifestPushServiceImpl{ logger: logger, - gitFactory: gitFactory, pipelineStatusTimelineService: pipelineStatusTimelineService, pipelineStatusTimelineRepository: pipelineStatusTimelineRepository, acdConfig: acdConfig, @@ -140,26 +134,14 @@ func (impl *GitOpsManifestPushServiceImpl) CommitValuesToGit(manifestPushTemplat UserName: userName, UserEmailId: userEmailId, } - bitbucketMetadata, err := impl.gitOpsConfigReadService.GetBitbucketMetadata() - if err != nil { - impl.logger.Errorw("error in getting bitbucket metadata", "err", err) - return commitHash, commitTime, err - } - gitOpsConfig := &bean2.GitOpsConfigDto{BitBucketWorkspaceId: bitbucketMetadata.BitBucketWorkspaceId} - _, span = otel.Tracer("orchestrator").Start(ctx, "gitFactory.Client.CommitValues") - commitHash, commitTime, err = impl.gitFactory.Client.CommitValues(chartGitAttr, gitOpsConfig) + + _, span = otel.Tracer("orchestrator").Start(ctx, "gitOpsRemoteOperationService.CommitValues") + commitHash, commitTime, err = impl.gitOpsRemoteOperationService.CommitValues(chartGitAttr) span.End() if err != nil { impl.logger.Errorw("error in git commit", "err", err) return commitHash, commitTime, err } - if commitTime.IsZero() { - commitTime = time.Now() - } - span.End() - if err != nil { - return commitHash, commitTime, err - } return commitHash, commitTime, nil } diff --git a/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go b/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go index ed86999b8b..08460991af 100644 --- a/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go +++ b/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go @@ -53,7 +53,6 @@ type AppStoreDeploymentCommonService interface { GetValuesAndRequirementGitConfig(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*util.ChartConfig, *util.ChartConfig, error) CreateChartProxyAndGetPath(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*util.ChartCreateResponse, error) CreateGitOpsRepoAndPushChart(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, builtChartPath string, requirementsConfig *util.ChartConfig, valuesConfig *util.ChartConfig) (*commonBean.ChartGitAttribute, bool, string, error) - CommitConfigToGit(chartConfig *util.ChartConfig) (gitHash string, err error) GetGitCommitConfig(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, fileString string, filename string) (*util.ChartConfig, error) GetValuesString(chartName, valuesOverrideYaml string) (string, error) GetRequirementsString(appStoreVersionId int) (string, error) @@ -404,7 +403,6 @@ func (impl AppStoreDeploymentCommonServiceImpl) GenerateManifest(installAppVersi // CreateGitOpsRepo creates a gitOps repo with readme func (impl AppStoreDeploymentCommonServiceImpl) CreateGitOpsRepo(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (string, bool, error) { - if len(installAppVersionRequest.GitOpsRepoName) == 0 { //here gitops repo will be the app name, to breaking the mono repo structure gitOpsRepoName := impl.gitOpsConfigReadService.GetGitOpsRepoName(installAppVersionRequest.AppName) @@ -416,21 +414,16 @@ func (impl AppStoreDeploymentCommonServiceImpl) CreateGitOpsRepo(installAppVersi return "", false, err } //getting user name & emailId for commit author data - userEmailId, userName := impl.gitOpsConfigReadService.GetUserEmailIdAndNameForGitOpsCommit(installAppVersionRequest.UserId) gitRepoRequest := &bean.GitOpsConfigDto{ GitRepoName: installAppVersionRequest.GitOpsRepoName, Description: "helm chart for " + installAppVersionRequest.GitOpsRepoName, - Username: userName, - UserEmailId: userEmailId, BitBucketWorkspaceId: bitbucketMetadata.BitBucketWorkspaceId, BitBucketProjectKey: bitbucketMetadata.BitBucketProjectKey, } - repoUrl, isNew, detailedError := impl.gitFactory.Client.CreateRepository(gitRepoRequest) - for _, err := range detailedError.StageErrorMap { - if err != nil { - impl.logger.Errorw("error in creating git project", "name", installAppVersionRequest.GitOpsRepoName, "err", err) - return "", false, err - } + repoUrl, isNew, err := impl.gitOpsRemoteOperationService.CreateRepository(gitRepoRequest, installAppVersionRequest.UserId) + if err != nil { + impl.logger.Errorw("error in creating git project", "name", installAppVersionRequest.GitOpsRepoName, "err", err) + return "", false, err } return repoUrl, isNew, err } @@ -540,22 +533,6 @@ func (impl AppStoreDeploymentCommonServiceImpl) CreateGitOpsRepoAndPushChart(ins return chartGitAttribute, isNew, commitHash, err } -// CommitConfigToGit is used for committing values.yaml and requirements.yaml file config -func (impl AppStoreDeploymentCommonServiceImpl) CommitConfigToGit(chartConfig *util.ChartConfig) (string, error) { - bitbucketMetadata, err := impl.gitOpsConfigReadService.GetBitbucketMetadata() - if err != nil { - impl.logger.Errorw("error in getting bitbucket metadata", "err", err) - return "", err - } - gitOpsConfig := &bean.GitOpsConfigDto{BitBucketWorkspaceId: bitbucketMetadata.BitBucketWorkspaceId} - gitHash, _, err := impl.gitFactory.Client.CommitValues(chartConfig, gitOpsConfig) - if err != nil { - impl.logger.Errorw("error in git commit", "err", err) - return "", err - } - return gitHash, nil -} - func (impl AppStoreDeploymentCommonServiceImpl) GitOpsOperations(manifestResponse *AppStoreManifestResponse, installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*AppStoreGitOpsResponse, error) { appStoreGitOpsResponse := &AppStoreGitOpsResponse{} chartGitAttribute, isNew, githash, err := impl.CreateGitOpsRepoAndPushChart(installAppVersionRequest, manifestResponse.ChartResponse.BuiltChartPath, manifestResponse.RequirementsConfig, manifestResponse.ValuesConfig) @@ -570,7 +547,7 @@ func (impl AppStoreDeploymentCommonServiceImpl) GitOpsOperations(manifestRespons // Checking this is the first time chart has been pushed , if yes requirements.yaml has been already pushed with chart as there was sync-delay with github api. // step-2 commit dependencies and values in git if !isNew { - _, err = impl.CommitConfigToGit(manifestResponse.RequirementsConfig) + _, _, err = impl.gitOpsRemoteOperationService.CommitValues(manifestResponse.RequirementsConfig) if err != nil { impl.logger.Errorw("error in committing dependency config to git", "err", err) return appStoreGitOpsResponse, err @@ -581,7 +558,7 @@ func (impl AppStoreDeploymentCommonServiceImpl) GitOpsOperations(manifestRespons return appStoreGitOpsResponse, err } - githash, err = impl.CommitConfigToGit(manifestResponse.ValuesConfig) + githash, _, err = impl.gitOpsRemoteOperationService.CommitValues(manifestResponse.ValuesConfig) if err != nil { impl.logger.Errorw("error in committing values config to git", "err", err) return appStoreGitOpsResponse, err diff --git a/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go b/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go index 6293aa0cb6..bbed6844f3 100644 --- a/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go +++ b/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go @@ -26,6 +26,7 @@ import ( "github.com/devtron-labs/common-lib/pubsub-lib/model" commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote" "time" "github.com/devtron-labs/devtron/client/argocdServer" @@ -73,6 +74,7 @@ type AppStoreDeploymentFullModeServiceImpl struct { installedAppRepositoryHistory repository4.InstalledAppVersionHistoryRepository ACDConfig *argocdServer.ACDConfig gitOpsConfigReadService config.GitOpsConfigReadService + gitOpsRemoteOperationService remote.GitOpsRemoteOperationService } func NewAppStoreDeploymentFullModeServiceImpl(logger *zap.SugaredLogger, @@ -84,7 +86,8 @@ func NewAppStoreDeploymentFullModeServiceImpl(logger *zap.SugaredLogger, pubSubClient *pubsub_lib.PubSubClientServiceImpl, installedAppRepositoryHistory repository4.InstalledAppVersionHistoryRepository, ACDConfig *argocdServer.ACDConfig, - gitOpsConfigReadService config.GitOpsConfigReadService) *AppStoreDeploymentFullModeServiceImpl { + gitOpsConfigReadService config.GitOpsConfigReadService, + gitOpsRemoteOperationService remote.GitOpsRemoteOperationService) *AppStoreDeploymentFullModeServiceImpl { appStoreDeploymentFullModeServiceImpl := &AppStoreDeploymentFullModeServiceImpl{ logger: logger, acdClient: acdClient, @@ -98,6 +101,7 @@ func NewAppStoreDeploymentFullModeServiceImpl(logger *zap.SugaredLogger, installedAppRepositoryHistory: installedAppRepositoryHistory, ACDConfig: ACDConfig, gitOpsConfigReadService: gitOpsConfigReadService, + gitOpsRemoteOperationService: gitOpsRemoteOperationService, } err := appStoreDeploymentFullModeServiceImpl.SubscribeHelmInstallStatus() if err != nil { @@ -226,7 +230,7 @@ func (impl AppStoreDeploymentFullModeServiceImpl) UpdateValuesYaml(installAppVer impl.logger.Errorw("error in getting git commit config", "err", err) } - commitHash, err := impl.appStoreDeploymentCommonService.CommitConfigToGit(valuesGitConfig) + commitHash, _, err := impl.gitOpsRemoteOperationService.CommitValues(valuesGitConfig) if err != nil { impl.logger.Errorw("error in git commit", "err", err) return installAppVersionRequest, errors.New(pipelineConfig.TIMELINE_STATUS_GIT_COMMIT_FAILED) @@ -250,7 +254,7 @@ func (impl AppStoreDeploymentFullModeServiceImpl) UpdateRequirementYaml(installA return err } - _, err = impl.appStoreDeploymentCommonService.CommitConfigToGit(requirementsGitConfig) + _, _, err = impl.gitOpsRemoteOperationService.CommitValues(requirementsGitConfig) if err != nil { impl.logger.Errorw("error in values commit", "err", err) return errors.New(pipelineConfig.TIMELINE_STATUS_GIT_COMMIT_FAILED) diff --git a/pkg/appStore/deployment/service/AppStoreDeploymentService.go b/pkg/appStore/deployment/service/AppStoreDeploymentService.go index 89c61c6d09..9ae90a3e2c 100644 --- a/pkg/appStore/deployment/service/AppStoreDeploymentService.go +++ b/pkg/appStore/deployment/service/AppStoreDeploymentService.go @@ -44,6 +44,7 @@ import ( cluster2 "github.com/devtron-labs/devtron/pkg/cluster" clusterRepository "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote" "github.com/devtron-labs/devtron/pkg/sql" util2 "github.com/devtron-labs/devtron/util" "github.com/go-pg/pg" @@ -108,6 +109,7 @@ type AppStoreDeploymentServiceImpl struct { deploymentTypeConfig *DeploymentServiceTypeConfig aCDConfig *argocdServer.ACDConfig gitOpsConfigReadService config.GitOpsConfigReadService + gitOpsRemoteOperationService remote.GitOpsRemoteOperationService } func NewAppStoreDeploymentServiceImpl(logger *zap.SugaredLogger, installedAppRepository repository.InstalledAppRepository, @@ -119,7 +121,8 @@ func NewAppStoreDeploymentServiceImpl(logger *zap.SugaredLogger, installedAppRep globalEnvVariables *util2.GlobalEnvVariables, installedAppRepositoryHistory repository.InstalledAppVersionHistoryRepository, gitOpsRepository repository2.GitOpsConfigRepository, deploymentTypeConfig *DeploymentServiceTypeConfig, aCDConfig *argocdServer.ACDConfig, - gitOpsConfigReadService config.GitOpsConfigReadService) *AppStoreDeploymentServiceImpl { + gitOpsConfigReadService config.GitOpsConfigReadService, + gitOpsRemoteOperationService remote.GitOpsRemoteOperationService) *AppStoreDeploymentServiceImpl { appStoreDeploymentServiceImpl := &AppStoreDeploymentServiceImpl{ logger: logger, @@ -141,6 +144,7 @@ func NewAppStoreDeploymentServiceImpl(logger *zap.SugaredLogger, installedAppRep deploymentTypeConfig: deploymentTypeConfig, aCDConfig: aCDConfig, gitOpsConfigReadService: gitOpsConfigReadService, + gitOpsRemoteOperationService: gitOpsRemoteOperationService, } return appStoreDeploymentServiceImpl } @@ -1474,12 +1478,12 @@ func (impl *AppStoreDeploymentServiceImpl) UpdateInstalledApp(ctx context.Contex } else if isChartChanged || isVersionChanged { // update dependency if chart or chart version is changed - _, requirementsCommitErr = impl.appStoreDeploymentCommonService.CommitConfigToGit(manifest.RequirementsConfig) - gitHash, valuesCommitErr = impl.appStoreDeploymentCommonService.CommitConfigToGit(manifest.ValuesConfig) + _, _, requirementsCommitErr = impl.gitOpsRemoteOperationService.CommitValues(manifest.RequirementsConfig) + gitHash, _, valuesCommitErr = impl.gitOpsRemoteOperationService.CommitValues(manifest.ValuesConfig) } else { // only values are changed in update, so commit values config - gitHash, valuesCommitErr = impl.appStoreDeploymentCommonService.CommitConfigToGit(manifest.ValuesConfig) + gitHash, _, valuesCommitErr = impl.gitOpsRemoteOperationService.CommitValues(manifest.ValuesConfig) } if valuesCommitErr != nil || requirementsCommitErr != nil { diff --git a/pkg/appStore/deployment/service/InstalledAppService.go b/pkg/appStore/deployment/service/InstalledAppService.go index eb1c89ad53..507b3a4974 100644 --- a/pkg/appStore/deployment/service/InstalledAppService.go +++ b/pkg/appStore/deployment/service/InstalledAppService.go @@ -22,6 +22,7 @@ import ( "context" commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote" /* #nosec */ "crypto/sha1" @@ -122,7 +123,6 @@ type InstalledAppServiceImpl struct { chartGroupDeploymentRepository repository2.ChartGroupDeploymentRepository envService cluster2.EnvironmentService ArgoK8sClient argocdServer.ArgoK8sClient - gitFactory *util.GitFactory aCDAuthConfig *util2.ACDAuthConfig gitOpsRepository repository3.GitOpsConfigRepository userService user.UserService @@ -140,6 +140,7 @@ type InstalledAppServiceImpl struct { k8sApplicationService application3.K8sApplicationService acdConfig *argocdServer.ACDConfig gitOpsConfigReadService config.GitOpsConfigReadService + gitOpsRemoteOperationService remote.GitOpsRemoteOperationService } func NewInstalledAppServiceImpl(logger *zap.SugaredLogger, @@ -154,7 +155,7 @@ func NewInstalledAppServiceImpl(logger *zap.SugaredLogger, tokenCache *util2.TokenCache, chartGroupDeploymentRepository repository2.ChartGroupDeploymentRepository, envService cluster2.EnvironmentService, argoK8sClient argocdServer.ArgoK8sClient, - gitFactory *util.GitFactory, aCDAuthConfig *util2.ACDAuthConfig, gitOpsRepository repository3.GitOpsConfigRepository, userService user.UserService, + aCDAuthConfig *util2.ACDAuthConfig, gitOpsRepository repository3.GitOpsConfigRepository, userService user.UserService, appStoreDeploymentFullModeService appStoreDeploymentFullMode.AppStoreDeploymentFullModeService, appStoreDeploymentService AppStoreDeploymentService, installedAppRepositoryHistory repository2.InstalledAppVersionHistoryRepository, @@ -163,8 +164,8 @@ func NewInstalledAppServiceImpl(logger *zap.SugaredLogger, pipelineStatusTimelineService status.PipelineStatusTimelineService, appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService, k8sCommonService k8s.K8sCommonService, k8sApplicationService application3.K8sApplicationService, - acdConfig *argocdServer.ACDConfig, - gitOpsConfigReadService config.GitOpsConfigReadService) (*InstalledAppServiceImpl, error) { + acdConfig *argocdServer.ACDConfig, gitOpsConfigReadService config.GitOpsConfigReadService, + gitOpsRemoteOperationService remote.GitOpsRemoteOperationService) (*InstalledAppServiceImpl, error) { impl := &InstalledAppServiceImpl{ logger: logger, installedAppRepository: installedAppRepository, @@ -180,7 +181,6 @@ func NewInstalledAppServiceImpl(logger *zap.SugaredLogger, chartGroupDeploymentRepository: chartGroupDeploymentRepository, envService: envService, ArgoK8sClient: argoK8sClient, - gitFactory: gitFactory, aCDAuthConfig: aCDAuthConfig, gitOpsRepository: gitOpsRepository, userService: userService, @@ -198,6 +198,7 @@ func NewInstalledAppServiceImpl(logger *zap.SugaredLogger, k8sApplicationService: k8sApplicationService, acdConfig: acdConfig, gitOpsConfigReadService: gitOpsConfigReadService, + gitOpsRemoteOperationService: gitOpsRemoteOperationService, } err := impl.Subscribe() if err != nil { @@ -475,21 +476,13 @@ func (impl InstalledAppServiceImpl) performDeployStageOnAcd(installedAppVersion impl.logger.Errorw("fetching error", "err", err) return nil, err } - bitbucketMetadata, err := impl.gitOpsConfigReadService.GetBitbucketMetadata() - if err != nil { - impl.logger.Errorw("error in getting bitbucket metadata", "err", err) - return nil, err - } - config := &bean2.GitOpsConfigDto{ - GitRepoName: installedAppVersion.GitOpsRepoName, - BitBucketWorkspaceId: bitbucketMetadata.BitBucketWorkspaceId, - BitBucketProjectKey: bitbucketMetadata.BitBucketProjectKey, - } - repoUrl, err := impl.gitFactory.Client.GetRepoUrl(config) + + repoUrl, err := impl.gitOpsRemoteOperationService.GetRepoUrlByRepoName(installedAppVersion.GitOpsRepoName) if err != nil { //will allow to continue to persist status on next operation - impl.logger.Errorw("fetching error", "err", err) + impl.logger.Errorw("error, GetRepoUrlByRepoName", "err", err) } + chartGitAttr.RepoUrl = repoUrl chartGitAttr.ChartLocation = fmt.Sprintf("%s-%s", installedAppVersion.AppName, environment.Name) installedAppVersion.ACDAppName = fmt.Sprintf("%s-%s", installedAppVersion.AppName, environment.Name) diff --git a/pkg/appStore/deployment/tool/AppStoreDeploymentHelmService.go b/pkg/appStore/deployment/tool/AppStoreDeploymentHelmService.go index c317353f48..520cf28025 100644 --- a/pkg/appStore/deployment/tool/AppStoreDeploymentHelmService.go +++ b/pkg/appStore/deployment/tool/AppStoreDeploymentHelmService.go @@ -5,6 +5,7 @@ import ( "errors" repository2 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote" "net/http" "time" @@ -49,10 +50,15 @@ type AppStoreDeploymentHelmServiceImpl struct { installedAppRepository repository.InstalledAppRepository appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService OCIRegistryConfigRepository repository2.OCIRegistryConfigRepository + gitOpsRemoteOperationService remote.GitOpsRemoteOperationService } -func NewAppStoreDeploymentHelmServiceImpl(logger *zap.SugaredLogger, helmAppService client.HelmAppService, appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository, - environmentRepository clusterRepository.EnvironmentRepository, helmAppClient client.HelmAppClient, installedAppRepository repository.InstalledAppRepository, appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService, OCIRegistryConfigRepository repository2.OCIRegistryConfigRepository) *AppStoreDeploymentHelmServiceImpl { +func NewAppStoreDeploymentHelmServiceImpl(logger *zap.SugaredLogger, helmAppService client.HelmAppService, + appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository, + environmentRepository clusterRepository.EnvironmentRepository, helmAppClient client.HelmAppClient, + installedAppRepository repository.InstalledAppRepository, appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService, + OCIRegistryConfigRepository repository2.OCIRegistryConfigRepository, + gitOpsRemoteOperationService remote.GitOpsRemoteOperationService) *AppStoreDeploymentHelmServiceImpl { return &AppStoreDeploymentHelmServiceImpl{ Logger: logger, helmAppService: helmAppService, @@ -62,6 +68,7 @@ func NewAppStoreDeploymentHelmServiceImpl(logger *zap.SugaredLogger, helmAppServ installedAppRepository: installedAppRepository, appStoreDeploymentCommonService: appStoreDeploymentCommonService, OCIRegistryConfigRepository: OCIRegistryConfigRepository, + gitOpsRemoteOperationService: gitOpsRemoteOperationService, } } @@ -322,7 +329,7 @@ func (impl *AppStoreDeploymentHelmServiceImpl) UpdateRequirementDependencies(ins impl.Logger.Errorw("error in getting git config for helm app", "err", err) return err } - _, err = impl.appStoreDeploymentCommonService.CommitConfigToGit(requirementsGitConfig) + _, _, err = impl.gitOpsRemoteOperationService.CommitValues(requirementsGitConfig) if err != nil { impl.Logger.Errorw("error in committing config to git for helm app", "err", err) return err @@ -347,7 +354,7 @@ func (impl *AppStoreDeploymentHelmServiceImpl) UpdateValuesDependencies(installA impl.Logger.Errorw("error in getting git config for helm app", "err", err) return err } - _, err = impl.appStoreDeploymentCommonService.CommitConfigToGit(valuesGitConfig) + _, _, err = impl.gitOpsRemoteOperationService.CommitValues(valuesGitConfig) if err != nil { impl.Logger.Errorw("error in committing config to git for helm app", "err", err) return err diff --git a/pkg/appStore/deployment/tool/gitops/AppStoreDeploymentArgoCdService.go b/pkg/appStore/deployment/tool/gitops/AppStoreDeploymentArgoCdService.go index 9bfcf39a40..972ed626ea 100644 --- a/pkg/appStore/deployment/tool/gitops/AppStoreDeploymentArgoCdService.go +++ b/pkg/appStore/deployment/tool/gitops/AppStoreDeploymentArgoCdService.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote" "net/http" "strings" "time" @@ -67,7 +68,6 @@ type AppStoreDeploymentArgoCdServiceImpl struct { chartGroupDeploymentRepository repository.ChartGroupDeploymentRepository installedAppRepository repository.InstalledAppRepository installedAppRepositoryHistory repository.InstalledAppVersionHistoryRepository - gitFactory *util.GitFactory argoUserService argo.ArgoUserService appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService helmAppService client.HelmAppService @@ -78,18 +78,19 @@ type AppStoreDeploymentArgoCdServiceImpl struct { appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository argoClientWrapperService argocdServer.ArgoClientWrapperService acdConfig *argocdServer.ACDConfig + gitOpsRemoteOperationService remote.GitOpsRemoteOperationService } func NewAppStoreDeploymentArgoCdServiceImpl(logger *zap.SugaredLogger, appStoreDeploymentFullModeService appStoreDeploymentFullMode.AppStoreDeploymentFullModeService, acdClient application2.ServiceClient, chartGroupDeploymentRepository repository.ChartGroupDeploymentRepository, installedAppRepository repository.InstalledAppRepository, installedAppRepositoryHistory repository.InstalledAppVersionHistoryRepository, - gitFactory *util.GitFactory, argoUserService argo.ArgoUserService, appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService, + argoUserService argo.ArgoUserService, appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService, helmAppService client.HelmAppService, appStatusService appStatus.AppStatusService, pipelineStatusTimelineService status.PipelineStatusTimelineService, userService user.UserService, pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository, appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository, argoClientWrapperService argocdServer.ArgoClientWrapperService, acdConfig *argocdServer.ACDConfig, -) *AppStoreDeploymentArgoCdServiceImpl { + gitOpsRemoteOperationService remote.GitOpsRemoteOperationService) *AppStoreDeploymentArgoCdServiceImpl { return &AppStoreDeploymentArgoCdServiceImpl{ Logger: logger, appStoreDeploymentFullModeService: appStoreDeploymentFullModeService, @@ -97,7 +98,6 @@ func NewAppStoreDeploymentArgoCdServiceImpl(logger *zap.SugaredLogger, appStoreD chartGroupDeploymentRepository: chartGroupDeploymentRepository, installedAppRepository: installedAppRepository, installedAppRepositoryHistory: installedAppRepositoryHistory, - gitFactory: gitFactory, argoUserService: argoUserService, appStoreDeploymentCommonService: appStoreDeploymentCommonService, helmAppService: helmAppService, @@ -108,6 +108,7 @@ func NewAppStoreDeploymentArgoCdServiceImpl(logger *zap.SugaredLogger, appStoreD appStoreApplicationVersionRepository: appStoreApplicationVersionRepository, argoClientWrapperService: argoClientWrapperService, acdConfig: acdConfig, + gitOpsRemoteOperationService: gitOpsRemoteOperationService, } } @@ -604,7 +605,7 @@ func (impl *AppStoreDeploymentArgoCdServiceImpl) UpdateRequirementDependencies(i impl.Logger.Errorw("error in getting git config for helm app", "err", err) return err } - _, err = impl.appStoreDeploymentCommonService.CommitConfigToGit(requirementsGitConfig) + _, _, err = impl.gitOpsRemoteOperationService.CommitValues(requirementsGitConfig) if err != nil { impl.Logger.Errorw("error in committing config to git for helm app", "err", err) return err @@ -697,7 +698,7 @@ func (impl AppStoreDeploymentArgoCdServiceImpl) updateValuesYaml(installAppVersi impl.Logger.Errorw("error in getting git config for helm app", "err", err) return nil, err } - gitHash, err := impl.appStoreDeploymentCommonService.CommitConfigToGit(valuesGitConfig) + gitHash, _, err := impl.gitOpsRemoteOperationService.CommitValues(valuesGitConfig) if err != nil { impl.Logger.Errorw("error in git commit", "err", err) _ = impl.SaveTimelineForACDHelmApps(installAppVersionRequest, pipelineConfig.TIMELINE_STATUS_GIT_COMMIT_FAILED, fmt.Sprintf("Git commit failed - %v", err), time.Now(), tx) diff --git a/pkg/deployment/gitOps/remote/GitOpsRemoteOperationService.go b/pkg/deployment/gitOps/remote/GitOpsRemoteOperationService.go index f956e862e8..18369cccf3 100644 --- a/pkg/deployment/gitOps/remote/GitOpsRemoteOperationService.go +++ b/pkg/deployment/gitOps/remote/GitOpsRemoteOperationService.go @@ -14,6 +14,7 @@ import ( "path/filepath" "regexp" "sigs.k8s.io/yaml" + "time" ) type GitOpsRemoteOperationService interface { @@ -25,6 +26,9 @@ type GitOpsRemoteOperationService interface { CreateChartProxy(chartMetaData *chart.Metadata, refChartLocation string, envName string, chartProxyReq *bean.ChartProxyReqDto) (string, *commonBean.ChartGitAttribute, error) GitPull(clonedDir string, repoUrl string, appStoreName string) error + CommitValues(chartGitAttr *util.ChartConfig) (commitHash string, commitTime time.Time, err error) + CreateRepository(dto *bean2.GitOpsConfigDto, userId int32) (string, bool, error) + GetRepoUrlByRepoName(repoName string) (string, error) } type GitOpsRemoteOperationServiceImpl struct { @@ -314,3 +318,58 @@ func (impl *GitOpsRemoteOperationServiceImpl) GitPull(clonedDir string, repoUrl } return nil } + +func (impl *GitOpsRemoteOperationServiceImpl) CommitValues(chartGitAttr *util.ChartConfig) (commitHash string, commitTime time.Time, err error) { + bitbucketMetadata, err := impl.gitOpsConfigReadService.GetBitbucketMetadata() + if err != nil { + impl.logger.Errorw("error in getting bitbucket metadata", "err", err) + return commitHash, commitTime, err + } + gitOpsConfig := &bean2.GitOpsConfigDto{BitBucketWorkspaceId: bitbucketMetadata.BitBucketWorkspaceId} + commitHash, commitTime, err = impl.gitFactory.Client.CommitValues(chartGitAttr, gitOpsConfig) + if err != nil { + impl.logger.Errorw("error in git commit", "err", err) + return commitHash, commitTime, err + } + if commitTime.IsZero() { + commitTime = time.Now() + } + return commitHash, commitTime, nil +} + +func (impl *GitOpsRemoteOperationServiceImpl) CreateRepository(dto *bean2.GitOpsConfigDto, userId int32) (string, bool, error) { + //getting user name & emailId for commit author data + userEmailId, userName := impl.gitOpsConfigReadService.GetUserEmailIdAndNameForGitOpsCommit(userId) + if dto != nil { + dto.UserEmailId = userEmailId + dto.Username = userName + } + repoUrl, isNew, detailedError := impl.gitFactory.Client.CreateRepository(dto) + for _, err := range detailedError.StageErrorMap { + if err != nil { + impl.logger.Errorw("error in creating git project", "err", err, "req", dto) + return "", false, err + } + } + return repoUrl, isNew, nil +} + +func (impl *GitOpsRemoteOperationServiceImpl) GetRepoUrlByRepoName(repoName string) (string, error) { + repoUrl := "" + bitbucketMetadata, err := impl.gitOpsConfigReadService.GetBitbucketMetadata() + if err != nil { + impl.logger.Errorw("error in getting bitbucket metadata", "err", err) + return repoUrl, err + } + dto := &bean2.GitOpsConfigDto{ + GitRepoName: repoName, + BitBucketWorkspaceId: bitbucketMetadata.BitBucketWorkspaceId, + BitBucketProjectKey: bitbucketMetadata.BitBucketProjectKey, + } + repoUrl, err = impl.gitFactory.Client.GetRepoUrl(dto) + if err != nil { + //will allow to continue to persist status on next operation + impl.logger.Errorw("error in getting repo url", "err", err, "repoName", repoName) + } + return repoUrl, nil +} diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index 1eebea4b9d..e3f72bebd3 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -23,6 +23,7 @@ import ( errors3 "errors" "fmt" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" "path" @@ -183,7 +184,6 @@ type WorkflowDagExecutorImpl struct { environmentConfigRepository chartConfig.EnvConfigOverrideRepository dbMigrationConfigRepository pipelineConfig.DbMigrationConfigRepository mergeUtil *util.MergeUtil - gitFactory *util.GitFactory acdClient application2.ServiceClient argoClientWrapperService argocdServer.ArgoClientWrapperService customTagService CustomTagService @@ -191,6 +191,7 @@ type WorkflowDagExecutorImpl struct { deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService chartRefService chartRef.ChartRefService gitOpsConfigReadService config.GitOpsConfigReadService + gitOpsRemoteOperationService remote.GitOpsRemoteOperationService } const kedaAutoscaling = "kedaAutoscaling" @@ -285,7 +286,6 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi environmentConfigRepository chartConfig.EnvConfigOverrideRepository, dbMigrationConfigRepository pipelineConfig.DbMigrationConfigRepository, mergeUtil *util.MergeUtil, - gitFactory *util.GitFactory, acdClient application2.ServiceClient, argoClientWrapperService argocdServer.ArgoClientWrapperService, pipelineConfigListenerService PipelineConfigListenerService, @@ -293,7 +293,8 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi ACDConfig *argocdServer.ACDConfig, deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService, chartRefService chartRef.ChartRefService, - gitOpsConfigReadService config.GitOpsConfigReadService) *WorkflowDagExecutorImpl { + gitOpsConfigReadService config.GitOpsConfigReadService, + gitOpsRemoteOperationService remote.GitOpsRemoteOperationService) *WorkflowDagExecutorImpl { wde := &WorkflowDagExecutorImpl{logger: Logger, pipelineRepository: pipelineRepository, cdWorkflowRepository: cdWorkflowRepository, @@ -355,7 +356,6 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi environmentConfigRepository: environmentConfigRepository, dbMigrationConfigRepository: dbMigrationConfigRepository, mergeUtil: mergeUtil, - gitFactory: gitFactory, acdClient: acdClient, argoClientWrapperService: argoClientWrapperService, customTagService: customTagService, @@ -363,6 +363,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi deployedAppMetricsService: deployedAppMetricsService, chartRefService: chartRefService, gitOpsConfigReadService: gitOpsConfigReadService, + gitOpsRemoteOperationService: gitOpsRemoteOperationService, } config, err := types.GetCdConfig() if err != nil { @@ -4139,7 +4140,7 @@ func (impl *WorkflowDagExecutorImpl) mergeAndSave(envOverride *chartConfig.EnvCo } commitHash := "" - commitTime := time.Time{} + commitTime := time.Now() //setting default to current time, if not acd type then will go as it is else will get overriden if util.IsAcdApp(pipeline.DeploymentAppType) { chartRepoName := impl.gitOpsConfigReadService.GetGitOpsRepoNameFromUrl(envOverride.Chart.GitRepoUrl) _, span = otel.Tracer("orchestrator").Start(ctx, "chartTemplateService.GetUserEmailIdAndNameForGitOpsCommit") @@ -4156,23 +4157,14 @@ func (impl *WorkflowDagExecutorImpl) mergeAndSave(envOverride *chartConfig.EnvCo UserName: userName, UserEmailId: userEmailId, } - bitbucketMetadata, err := impl.gitOpsConfigReadService.GetBitbucketMetadata() - if err != nil { - impl.logger.Errorw("error in getting bitbucket metadata", "err", err) - return 0, 0, "", err - } - gitOpsConfig := &bean.GitOpsConfigDto{BitBucketWorkspaceId: bitbucketMetadata.BitBucketWorkspaceId} - _, span = otel.Tracer("orchestrator").Start(ctx, "gitFactory.Client.CommitValues") - commitHash, commitTime, err = impl.gitFactory.Client.CommitValues(chartGitAttr, gitOpsConfig) + _, span = otel.Tracer("orchestrator").Start(ctx, "gitOpsRemoteOperationService.CommitValues") + commitHash, commitTime, err = impl.gitOpsRemoteOperationService.CommitValues(chartGitAttr) span.End() if err != nil { impl.logger.Errorw("error in git commit", "err", err) return 0, 0, "", err } } - if commitTime.IsZero() { - commitTime = time.Now() - } pipelineOverride := &chartConfig.PipelineOverride{ Id: override.Id, GitHash: commitHash, diff --git a/wire_gen.go b/wire_gen.go index 420e078477..443233e933 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -436,7 +436,7 @@ func InitializeApp() (*App, error) { pipelineStrategyHistoryRepositoryImpl := repository8.NewPipelineStrategyHistoryRepositoryImpl(sugaredLogger, db) pipelineStrategyHistoryServiceImpl := history.NewPipelineStrategyHistoryServiceImpl(sugaredLogger, pipelineStrategyHistoryRepositoryImpl, userServiceImpl) manifestPushConfigRepositoryImpl := repository9.NewManifestPushConfigRepository(sugaredLogger, db) - gitOpsManifestPushServiceImpl := app2.NewGitOpsManifestPushServiceImpl(sugaredLogger, gitFactory, pipelineStatusTimelineServiceImpl, pipelineStatusTimelineRepositoryImpl, acdConfig, chartRefServiceImpl, gitOpsConfigReadServiceImpl, gitOpsRemoteOperationServiceImpl) + gitOpsManifestPushServiceImpl := app2.NewGitOpsManifestPushServiceImpl(sugaredLogger, pipelineStatusTimelineServiceImpl, pipelineStatusTimelineRepositoryImpl, acdConfig, chartRefServiceImpl, gitOpsConfigReadServiceImpl, gitOpsRemoteOperationServiceImpl) imageScanHistoryRepositoryImpl := security.NewImageScanHistoryRepositoryImpl(db, sugaredLogger) imageScanDeployInfoRepositoryImpl := security.NewImageScanDeployInfoRepositoryImpl(db, sugaredLogger) genericNoteRepositoryImpl := repository12.NewGenericNoteRepositoryImpl(db) @@ -452,7 +452,7 @@ func InitializeApp() (*App, error) { repositoryServiceClientImpl := repository13.NewServiceClientImpl(sugaredLogger, argoCDConnectionManagerImpl) argoClientWrapperServiceImpl := argocdServer.NewArgoClientWrapperServiceImpl(sugaredLogger, applicationServiceClientImpl, acdConfig, repositoryServiceClientImpl) pipelineConfigListenerServiceImpl := pipeline.NewPipelineConfigListenerServiceImpl(sugaredLogger) - workflowDagExecutorImpl := pipeline.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, pubSubClientServiceImpl, appServiceImpl, workflowServiceImpl, ciArtifactRepositoryImpl, ciPipelineRepositoryImpl, materialRepositoryImpl, pipelineOverrideRepositoryImpl, userServiceImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, enforcerUtilImpl, eventSimpleFactoryImpl, eventRESTClientImpl, cvePolicyRepositoryImpl, imageScanResultRepositoryImpl, appWorkflowRepositoryImpl, prePostCdScriptHistoryServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineServiceImpl, ciTemplateRepositoryImpl, ciWorkflowRepositoryImpl, appLabelRepositoryImpl, clientImpl, pipelineStageServiceImpl, k8sCommonServiceImpl, globalPluginServiceImpl, pluginInputVariableParserImpl, scopedVariableCMCSManagerImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineStrategyHistoryServiceImpl, manifestPushConfigRepositoryImpl, gitOpsManifestPushServiceImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, appCrudOperationServiceImpl, pipelineConfigRepositoryImpl, dockerRegistryIpsConfigServiceImpl, chartRepositoryImpl, chartTemplateServiceImpl, pipelineStrategyHistoryRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, argoK8sClientImpl, configMapRepositoryImpl, configMapHistoryRepositoryImpl, helmAppServiceImpl, helmAppClientImpl, envConfigOverrideRepositoryImpl, dbMigrationConfigRepositoryImpl, mergeUtil, gitFactory, applicationServiceClientImpl, argoClientWrapperServiceImpl, pipelineConfigListenerServiceImpl, customTagServiceImpl, acdConfig, deployedAppMetricsServiceImpl, chartRefServiceImpl, gitOpsConfigReadServiceImpl) + workflowDagExecutorImpl := pipeline.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, pubSubClientServiceImpl, appServiceImpl, workflowServiceImpl, ciArtifactRepositoryImpl, ciPipelineRepositoryImpl, materialRepositoryImpl, pipelineOverrideRepositoryImpl, userServiceImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, enforcerUtilImpl, eventSimpleFactoryImpl, eventRESTClientImpl, cvePolicyRepositoryImpl, imageScanResultRepositoryImpl, appWorkflowRepositoryImpl, prePostCdScriptHistoryServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineServiceImpl, ciTemplateRepositoryImpl, ciWorkflowRepositoryImpl, appLabelRepositoryImpl, clientImpl, pipelineStageServiceImpl, k8sCommonServiceImpl, globalPluginServiceImpl, pluginInputVariableParserImpl, scopedVariableCMCSManagerImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineStrategyHistoryServiceImpl, manifestPushConfigRepositoryImpl, gitOpsManifestPushServiceImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, appCrudOperationServiceImpl, pipelineConfigRepositoryImpl, dockerRegistryIpsConfigServiceImpl, chartRepositoryImpl, chartTemplateServiceImpl, pipelineStrategyHistoryRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, argoK8sClientImpl, configMapRepositoryImpl, configMapHistoryRepositoryImpl, helmAppServiceImpl, helmAppClientImpl, envConfigOverrideRepositoryImpl, dbMigrationConfigRepositoryImpl, mergeUtil, applicationServiceClientImpl, argoClientWrapperServiceImpl, pipelineConfigListenerServiceImpl, customTagServiceImpl, acdConfig, deployedAppMetricsServiceImpl, chartRefServiceImpl, gitOpsConfigReadServiceImpl, gitOpsRemoteOperationServiceImpl) deploymentGroupAppRepositoryImpl := repository.NewDeploymentGroupAppRepositoryImpl(sugaredLogger, db) deploymentGroupServiceImpl := deploymentGroup.NewDeploymentGroupServiceImpl(appRepositoryImpl, sugaredLogger, pipelineRepositoryImpl, ciPipelineRepositoryImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, deploymentGroupAppRepositoryImpl, ciArtifactRepositoryImpl, appWorkflowRepositoryImpl, workflowDagExecutorImpl) deploymentConfigServiceImpl := pipeline.NewDeploymentConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, pipelineRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, configMapHistoryServiceImpl, scopedVariableCMCSManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) @@ -545,15 +545,15 @@ func InitializeApp() (*App, error) { tokenCache := util3.NewTokenCache(sugaredLogger, acdAuthConfig, userAuthServiceImpl) chartGroupDeploymentRepositoryImpl := repository3.NewChartGroupDeploymentRepositoryImpl(db, sugaredLogger) appStoreDeploymentCommonServiceImpl := appStoreDeploymentCommon.NewAppStoreDeploymentCommonServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, chartTemplateServiceImpl, gitFactory, gitOpsConfigReadServiceImpl, gitOpsRemoteOperationServiceImpl) - appStoreDeploymentFullModeServiceImpl := appStoreDeploymentFullMode.NewAppStoreDeploymentFullModeServiceImpl(sugaredLogger, applicationServiceClientImpl, argoK8sClientImpl, acdAuthConfig, argoUserServiceImpl, pipelineStatusTimelineServiceImpl, appStoreDeploymentCommonServiceImpl, argoClientWrapperServiceImpl, pubSubClientServiceImpl, installedAppVersionHistoryRepositoryImpl, acdConfig, gitOpsConfigReadServiceImpl) + appStoreDeploymentFullModeServiceImpl := appStoreDeploymentFullMode.NewAppStoreDeploymentFullModeServiceImpl(sugaredLogger, applicationServiceClientImpl, argoK8sClientImpl, acdAuthConfig, argoUserServiceImpl, pipelineStatusTimelineServiceImpl, appStoreDeploymentCommonServiceImpl, argoClientWrapperServiceImpl, pubSubClientServiceImpl, installedAppVersionHistoryRepositoryImpl, acdConfig, gitOpsConfigReadServiceImpl, gitOpsRemoteOperationServiceImpl) clusterInstalledAppsRepositoryImpl := repository3.NewClusterInstalledAppsRepositoryImpl(db, sugaredLogger) - appStoreDeploymentHelmServiceImpl := appStoreDeploymentTool.NewAppStoreDeploymentHelmServiceImpl(sugaredLogger, helmAppServiceImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, helmAppClientImpl, installedAppRepositoryImpl, appStoreDeploymentCommonServiceImpl, ociRegistryConfigRepositoryImpl) - appStoreDeploymentArgoCdServiceImpl := appStoreDeploymentGitopsTool.NewAppStoreDeploymentArgoCdServiceImpl(sugaredLogger, appStoreDeploymentFullModeServiceImpl, applicationServiceClientImpl, chartGroupDeploymentRepositoryImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, gitFactory, argoUserServiceImpl, appStoreDeploymentCommonServiceImpl, helmAppServiceImpl, appStatusServiceImpl, pipelineStatusTimelineServiceImpl, userServiceImpl, pipelineStatusTimelineRepositoryImpl, appStoreApplicationVersionRepositoryImpl, argoClientWrapperServiceImpl, acdConfig) + appStoreDeploymentHelmServiceImpl := appStoreDeploymentTool.NewAppStoreDeploymentHelmServiceImpl(sugaredLogger, helmAppServiceImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, helmAppClientImpl, installedAppRepositoryImpl, appStoreDeploymentCommonServiceImpl, ociRegistryConfigRepositoryImpl, gitOpsRemoteOperationServiceImpl) + appStoreDeploymentArgoCdServiceImpl := appStoreDeploymentGitopsTool.NewAppStoreDeploymentArgoCdServiceImpl(sugaredLogger, appStoreDeploymentFullModeServiceImpl, applicationServiceClientImpl, chartGroupDeploymentRepositoryImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, appStoreDeploymentCommonServiceImpl, helmAppServiceImpl, appStatusServiceImpl, pipelineStatusTimelineServiceImpl, userServiceImpl, pipelineStatusTimelineRepositoryImpl, appStoreApplicationVersionRepositoryImpl, argoClientWrapperServiceImpl, acdConfig, gitOpsRemoteOperationServiceImpl) serviceDeploymentServiceTypeConfig, err := service2.GetDeploymentServiceTypeConfig() if err != nil { return nil, err } - appStoreDeploymentServiceImpl := service2.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, clusterInstalledAppsRepositoryImpl, appRepositoryImpl, appStoreDeploymentHelmServiceImpl, appStoreDeploymentArgoCdServiceImpl, environmentServiceImpl, clusterServiceImplExtended, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, globalEnvVariables, installedAppVersionHistoryRepositoryImpl, gitOpsConfigRepositoryImpl, serviceDeploymentServiceTypeConfig, acdConfig, gitOpsConfigReadServiceImpl) + appStoreDeploymentServiceImpl := service2.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, clusterInstalledAppsRepositoryImpl, appRepositoryImpl, appStoreDeploymentHelmServiceImpl, appStoreDeploymentArgoCdServiceImpl, environmentServiceImpl, clusterServiceImplExtended, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, globalEnvVariables, installedAppVersionHistoryRepositoryImpl, gitOpsConfigRepositoryImpl, serviceDeploymentServiceTypeConfig, acdConfig, gitOpsConfigReadServiceImpl, gitOpsRemoteOperationServiceImpl) k8sResourceHistoryRepositoryImpl := repository15.NewK8sResourceHistoryRepositoryImpl(db, sugaredLogger) k8sResourceHistoryServiceImpl := kubernetesResourceAuditLogs.Newk8sResourceHistoryServiceImpl(k8sResourceHistoryRepositoryImpl, sugaredLogger, appRepositoryImpl, environmentRepositoryImpl) ephemeralContainersRepositoryImpl := repository2.NewEphemeralContainersRepositoryImpl(db) @@ -563,7 +563,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - installedAppServiceImpl, err := service2.NewInstalledAppServiceImpl(sugaredLogger, installedAppRepositoryImpl, repositoryServiceClientImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, teamRepositoryImpl, appRepositoryImpl, applicationServiceClientImpl, appStoreValuesServiceImpl, pubSubClientServiceImpl, tokenCache, chartGroupDeploymentRepositoryImpl, environmentServiceImpl, argoK8sClientImpl, gitFactory, acdAuthConfig, gitOpsConfigRepositoryImpl, userServiceImpl, appStoreDeploymentFullModeServiceImpl, appStoreDeploymentServiceImpl, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, helmAppClientImpl, helmAppServiceImpl, appStatusServiceImpl, k8sUtil, pipelineStatusTimelineServiceImpl, appStoreDeploymentCommonServiceImpl, k8sCommonServiceImpl, k8sApplicationServiceImpl, acdConfig, gitOpsConfigReadServiceImpl) + installedAppServiceImpl, err := service2.NewInstalledAppServiceImpl(sugaredLogger, installedAppRepositoryImpl, repositoryServiceClientImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, teamRepositoryImpl, appRepositoryImpl, applicationServiceClientImpl, appStoreValuesServiceImpl, pubSubClientServiceImpl, tokenCache, chartGroupDeploymentRepositoryImpl, environmentServiceImpl, argoK8sClientImpl, acdAuthConfig, gitOpsConfigRepositoryImpl, userServiceImpl, appStoreDeploymentFullModeServiceImpl, appStoreDeploymentServiceImpl, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, helmAppClientImpl, helmAppServiceImpl, appStatusServiceImpl, k8sUtil, pipelineStatusTimelineServiceImpl, appStoreDeploymentCommonServiceImpl, k8sCommonServiceImpl, k8sApplicationServiceImpl, acdConfig, gitOpsConfigReadServiceImpl, gitOpsRemoteOperationServiceImpl) if err != nil { return nil, err } From f91f682d2a20ff2574ec21c7292e5fe44b14315f Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Thu, 18 Jan 2024 15:45:54 +0530 Subject: [PATCH 25/64] chore: AppStoreDeployment Install flow refactoring --- pkg/appStore/adapter/Adapter.go | 160 ++++++++++ pkg/appStore/bean/bean.go | 26 +- .../common/AppStoreDeploymentCommonService.go | 57 +--- .../repository/InstalledAppModels.go | 69 +++++ .../repository/InstalledAppRepository.go | 143 ++++----- .../repository/InstalledAppVersionHistory.go | 12 + .../service/AppStoreDeploymentDBService.go | 180 +++++++++++ .../service/AppStoreDeploymentService.go | 286 +----------------- .../service/AppStoreDeploymentService_test.go | 6 +- .../deployment/service/InstalledAppService.go | 4 +- .../service/InstalledAppService_test.go | 18 +- pkg/appStore/deployment/service/Notes.go | 109 ++++--- pkg/sql/UtilStructs.go | 14 + 13 files changed, 598 insertions(+), 486 deletions(-) create mode 100644 pkg/appStore/adapter/Adapter.go create mode 100644 pkg/appStore/deployment/repository/InstalledAppModels.go create mode 100644 pkg/appStore/deployment/service/AppStoreDeploymentDBService.go diff --git a/pkg/appStore/adapter/Adapter.go b/pkg/appStore/adapter/Adapter.go new file mode 100644 index 0000000000..4065e755d0 --- /dev/null +++ b/pkg/appStore/adapter/Adapter.go @@ -0,0 +1,160 @@ +package adapter + +import ( + "encoding/json" + "fmt" + "github.com/devtron-labs/devtron/internal/util" + appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" + "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" +) + +// NewInstallAppModel is used to generate new repository.InstalledApps model to be saved; +// Note: Do not use for update operations +func NewInstallAppModel(chart *appStoreBean.InstallAppVersionDTO, status appStoreBean.AppstoreDeploymentStatus) *repository.InstalledApps { + installAppModel := &repository.InstalledApps{ + AppId: chart.AppId, + EnvironmentId: chart.EnvironmentId, + DeploymentAppType: chart.DeploymentAppType, + } + if status != appStoreBean.WF_UNKNOWN { + installAppModel.UpdateStatus(status) + } + installAppModel.CreateAuditLog(chart.UserId) + installAppModel.UpdateGitOpsRepoName(chart.GitOpsRepoName) + installAppModel.MarkActive() + return installAppModel +} + +// NewInstallAppVersionsModel is used to generate new repository.InstalledAppVersions model to be saved; +// Note: Do not use for update operations +func NewInstallAppVersionsModel(chart *appStoreBean.InstallAppVersionDTO) *repository.InstalledAppVersions { + installedAppVersions := &repository.InstalledAppVersions{ + InstalledAppId: chart.InstalledAppId, + AppStoreApplicationVersionId: chart.AppStoreVersion, + ValuesYaml: chart.ValuesOverrideYaml, + ReferenceValueId: chart.ReferenceValueId, + ReferenceValueKind: chart.ReferenceValueKind, + } + installedAppVersions.CreateAuditLog(chart.UserId) + installedAppVersions.MarkActive() + return installedAppVersions +} + +// NewInstallAppVersionHistoryModel is used to generate new repository.InstalledAppVersionHistory model to be saved; +// Note: Do not use for update operations +func NewInstallAppVersionHistoryModel(chart *appStoreBean.InstallAppVersionDTO, status string, helmInstallConfigDTO appStoreBean.HelmReleaseStatusConfig) (*repository.InstalledAppVersionHistory, error) { + installedAppVersions := &repository.InstalledAppVersionHistory{ + InstalledAppVersionId: chart.InstalledAppVersionId, + ValuesYamlRaw: chart.ValuesOverrideYaml, + } + helmInstallConfig, err := json.Marshal(helmInstallConfigDTO) + if err != nil { + return nil, err + } + installedAppVersions.HelmReleaseStatusConfig = string(helmInstallConfig) + installedAppVersions.SetStartedOn() + installedAppVersions.SetStatus(status) + installedAppVersions.CreateAuditLog(chart.UserId) + return installedAppVersions, nil +} + +// NewClusterInstalledAppsModel is used to generate new repository.ClusterInstalledApps model to be saved; +// Note: Do not use for update operations +func NewClusterInstalledAppsModel(chart *appStoreBean.InstallAppVersionDTO, clusterId int) *repository.ClusterInstalledApps { + clusterInstalledAppsModel := &repository.ClusterInstalledApps{ + ClusterId: clusterId, + InstalledAppId: chart.InstalledAppId, + } + clusterInstalledAppsModel.CreateAuditLog(chart.UserId) + return clusterInstalledAppsModel +} + +// NewInstalledAppDeploymentAction is used to generate appStoreBean.InstalledAppDeploymentAction from deploymentAppType +func NewInstalledAppDeploymentAction(deploymentAppType string) *appStoreBean.InstalledAppDeploymentAction { + installedAppDeploymentAction := &appStoreBean.InstalledAppDeploymentAction{} + switch deploymentAppType { + case util.PIPELINE_DEPLOYMENT_TYPE_ACD: + installedAppDeploymentAction.PerformGitOps = true + installedAppDeploymentAction.PerformACDDeployment = true + installedAppDeploymentAction.PerformHelmDeployment = false + case util.PIPELINE_DEPLOYMENT_TYPE_HELM: + installedAppDeploymentAction.PerformGitOps = false + installedAppDeploymentAction.PerformACDDeployment = false + installedAppDeploymentAction.PerformHelmDeployment = true + case util.PIPELINE_DEPLOYMENT_TYPE_MANIFEST_DOWNLOAD: + installedAppDeploymentAction.PerformGitOps = false + installedAppDeploymentAction.PerformHelmDeployment = false + installedAppDeploymentAction.PerformACDDeployment = false + } + return installedAppDeploymentAction +} + +// GenerateInstallAppVersionDTO converts repository.InstalledApps and repository.InstalledAppVersions db object to appStoreBean.InstallAppVersionDTO bean +func GenerateInstallAppVersionDTO(chart *repository.InstalledApps, installedAppVersion *repository.InstalledAppVersions) *appStoreBean.InstallAppVersionDTO { + chartVersionApp := installedAppVersion.AppStoreApplicationVersion + + var chartRepoName, chartRepoUrl, Username, Password string + if chartVersionApp.AppStore.ChartRepoId != 0 { + chartRepo := chartVersionApp.AppStore.ChartRepo + chartRepoName = chartRepo.Name + chartRepoUrl = chartRepo.Url + Username = chartRepo.UserName + Password = chartRepo.Password + } else { + chartRepo := chartVersionApp.AppStore.DockerArtifactStore + chartRepoName = chartRepo.Id + chartRepoUrl = fmt.Sprintf("%s://%s/%s", + "oci", + chartVersionApp.AppStore.DockerArtifactStore.RegistryURL, + chartVersionApp.AppStore.Name) + Username = chartVersionApp.AppStore.DockerArtifactStore.Username + Password = chartVersionApp.AppStore.DockerArtifactStore.Password + } + + return &appStoreBean.InstallAppVersionDTO{ + EnvironmentId: chart.EnvironmentId, + AppId: chart.AppId, + TeamId: chart.App.TeamId, + TeamName: chart.App.Team.Name, + AppOfferingMode: chart.App.AppOfferingMode, + ClusterId: chart.Environment.ClusterId, + Namespace: chart.Environment.Namespace, + AppName: chart.App.AppName, + EnvironmentName: chart.Environment.Name, + InstalledAppId: chart.Id, + DeploymentAppType: chart.DeploymentAppType, + + Id: installedAppVersion.Id, + InstalledAppVersionId: installedAppVersion.Id, + InstallAppVersionChartDTO: &appStoreBean.InstallAppVersionChartDTO{ + AppStoreChartId: chartVersionApp.AppStore.Id, + ChartName: chartVersionApp.Name, + ChartVersion: chartVersionApp.Version, + InstallAppVersionChartRepoDTO: &appStoreBean.InstallAppVersionChartRepoDTO{ + RepoName: chartRepoName, + RepoUrl: chartRepoUrl, + UserName: Username, + Password: Password, + }, + }, + AppStoreApplicationVersionId: installedAppVersion.AppStoreApplicationVersionId, + } +} + +// GenerateInstallAppVersionMinDTO converts repository.InstalledApps db object to appStoreBean.InstallAppVersionDTO bean; +// Note: It only generates a minimal DTO and doesn't include repository.InstalledAppVersions data +func GenerateInstallAppVersionMinDTO(chart *repository.InstalledApps) *appStoreBean.InstallAppVersionDTO { + return &appStoreBean.InstallAppVersionDTO{ + EnvironmentId: chart.EnvironmentId, + InstalledAppId: chart.Id, + AppId: chart.AppId, + AppOfferingMode: chart.App.AppOfferingMode, + ClusterId: chart.Environment.ClusterId, + Namespace: chart.Environment.Namespace, + AppName: chart.App.AppName, + EnvironmentName: chart.Environment.Name, + TeamId: chart.App.TeamId, + TeamName: chart.App.Team.Name, + DeploymentAppType: chart.DeploymentAppType, + } +} diff --git a/pkg/appStore/bean/bean.go b/pkg/appStore/bean/bean.go index 314e0fd7f6..767777c2c1 100644 --- a/pkg/appStore/bean/bean.go +++ b/pkg/appStore/bean/bean.go @@ -63,7 +63,7 @@ type InstalledAppDto struct { } type InstallAppVersionDTO struct { - Id int `json:"id,omitempty"` + Id int `json:"id,omitempty"` // TODO: redundant data; refers to InstalledAppVersionId AppId int `json:"appId,omitempty"` AppName string `json:"appName,omitempty"` TeamId int `json:"teamId,omitempty"` @@ -76,8 +76,8 @@ type InstallAppVersionDTO struct { ValuesOverrideYaml string `json:"valuesOverrideYaml,omitempty"` Readme string `json:"readme,omitempty"` UserId int32 `json:"-"` - ReferenceValueId int `json:"referenceValueId, omitempty" validate:"required,number"` - ReferenceValueKind string `json:"referenceValueKind, omitempty" validate:"oneof=DEFAULT TEMPLATE DEPLOYED EXISTING"` + ReferenceValueId int `json:"referenceValueId, omitempty" validate:"required,number"` // TODO: ineffective usage of omitempty; can be removed + ReferenceValueKind string `json:"referenceValueKind, omitempty" validate:"oneof=DEFAULT TEMPLATE DEPLOYED EXISTING"` // TODO: ineffective usage of omitempty; can be removed ACDAppName string `json:"-"` Environment *repository2.Environment `json:"-"` ChartGroupEntryId int `json:"-"` @@ -96,14 +96,22 @@ type InstallAppVersionDTO struct { GitHash string `json:"gitHash"` EnvironmentName string `json:"-"` InstallAppVersionChartDTO *InstallAppVersionChartDTO `json:"-"` - DeploymentAppType string `json:"deploymentAppType"` + DeploymentAppType string `json:"deploymentAppType"` // TODO: instead of string, use enum AcdPartialDelete bool `json:"acdPartialDelete"` InstalledAppDeleteResponse *InstalledAppDeleteResponseDTO `json:"deleteResponse,omitempty"` AppStoreApplicationVersionId int - PerformGitOpsForHelmApp bool `json:"performGitOpsForHelmApp"` - PerformGitOps bool `json:"performGitOps"` - PerformACDDeployment bool `json:"performACDDeployment"` - PerformHelmDeployment bool `json:"performHelmDeployment"` +} + +func (chart *InstallAppVersionDTO) UpdateDeploymentAppType(deploymentAppType string) { + chart.DeploymentAppType = deploymentAppType +} + +// InstalledAppDeploymentAction is an internal struct for Helm App deployment; used to decide the deployment steps to be performed +type InstalledAppDeploymentAction struct { + PerformGitOpsForHelmApp bool + PerformGitOps bool + PerformACDDeployment bool + PerformHelmDeployment bool } type InstalledAppDeleteResponseDTO struct { @@ -144,7 +152,7 @@ type InstalledAppsResponse struct { EnvironmentName string `json:"environmentName"` DeployedAt time.Time `json:"deployedAt"` DeployedBy string `json:"deployedBy"` - DeploymentAppType string `json:"deploymentAppType,omitempty"` + DeploymentAppType string `json:"deploymentAppType,omitempty"` // TODO: instead of string, use enum InstalledAppsId int `json:"installedAppId"` Readme string `json:"readme"` EnvironmentId int `json:"environmentId"` diff --git a/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go b/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go index 44ca59836e..0a63b9420f 100644 --- a/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go +++ b/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go @@ -23,6 +23,7 @@ import ( "github.com/devtron-labs/devtron/api/bean" repository3 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/appStore/adapter" appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" appStoreDiscoverRepository "github.com/devtron-labs/devtron/pkg/appStore/discover/repository" @@ -141,7 +142,7 @@ func (impl AppStoreDeploymentCommonServiceImpl) GetInstalledAppByClusterNamespac if err != nil { return nil, err } - return impl.convert(installedApp, installedAppVersion), nil + return adapter.GenerateInstallAppVersionDTO(installedApp, installedAppVersion), nil } return nil, nil @@ -153,59 +154,7 @@ func (impl AppStoreDeploymentCommonServiceImpl) GetInstalledAppByInstalledAppId( return nil, err } installedApp := &installedAppVersion.InstalledApp - return impl.convert(installedApp, installedAppVersion), nil -} - -// converts db object to bean -func (impl AppStoreDeploymentCommonServiceImpl) convert(chart *repository.InstalledApps, installedAppVersion *repository.InstalledAppVersions) *appStoreBean.InstallAppVersionDTO { - - chartVersionApp := installedAppVersion.AppStoreApplicationVersion - - var chartRepoName, chartRepoUrl, Username, Password string - if installedAppVersion.AppStoreApplicationVersion.AppStore.ChartRepoId != 0 { - chartRepo := installedAppVersion.AppStoreApplicationVersion.AppStore.ChartRepo - chartRepoName = chartRepo.Name - chartRepoUrl = chartRepo.Url - Username = chartRepo.UserName - Password = chartRepo.Password - } else { - chartRepo := installedAppVersion.AppStoreApplicationVersion.AppStore.DockerArtifactStore - chartRepoName = chartRepo.Id - chartRepoUrl = fmt.Sprintf("%s://%s/%s", - "oci", - installedAppVersion.AppStoreApplicationVersion.AppStore.DockerArtifactStore.RegistryURL, - installedAppVersion.AppStoreApplicationVersion.AppStore.Name) - Username = installedAppVersion.AppStoreApplicationVersion.AppStore.DockerArtifactStore.Username - Password = installedAppVersion.AppStoreApplicationVersion.AppStore.DockerArtifactStore.Password - } - - return &appStoreBean.InstallAppVersionDTO{ - EnvironmentId: chart.EnvironmentId, - Id: chart.Id, - AppId: chart.AppId, - TeamId: chart.App.TeamId, - TeamName: chart.App.Team.Name, - AppOfferingMode: chart.App.AppOfferingMode, - ClusterId: chart.Environment.ClusterId, - Namespace: chart.Environment.Namespace, - AppName: chart.App.AppName, - EnvironmentName: chart.Environment.Name, - InstalledAppId: chart.Id, - InstalledAppVersionId: installedAppVersion.Id, - InstallAppVersionChartDTO: &appStoreBean.InstallAppVersionChartDTO{ - AppStoreChartId: chartVersionApp.AppStore.Id, - ChartName: chartVersionApp.Name, - ChartVersion: chartVersionApp.Version, - InstallAppVersionChartRepoDTO: &appStoreBean.InstallAppVersionChartRepoDTO{ - RepoName: chartRepoName, - RepoUrl: chartRepoUrl, - UserName: Username, - Password: Password, - }, - }, - DeploymentAppType: chart.DeploymentAppType, - AppStoreApplicationVersionId: installedAppVersion.AppStoreApplicationVersionId, - } + return adapter.GenerateInstallAppVersionDTO(installedApp, installedAppVersion), nil } func (impl AppStoreDeploymentCommonServiceImpl) ParseGitRepoErrorResponse(err error) (bool, error) { diff --git a/pkg/appStore/deployment/repository/InstalledAppModels.go b/pkg/appStore/deployment/repository/InstalledAppModels.go new file mode 100644 index 0000000000..56ea99d0d7 --- /dev/null +++ b/pkg/appStore/deployment/repository/InstalledAppModels.go @@ -0,0 +1,69 @@ +package repository + +import ( + "time" +) + +// TODO: Remove dependencies on native queries and the structs used in it + +// GitOpsAppDetails is used to operate on native query; This should be avoided. +// Usage func: InstalledAppRepository.GetAllGitOpsAppNameAndInstalledAppMapping +type GitOpsAppDetails struct { + GitOpsAppName string `sql:"git_ops_app_name"` + InstalledAppId int `sql:"installed_app_id"` +} + +// InstalledAppsWithChartDetails is used to operate on native query; This should be avoided. +// Usage func: InstalledAppRepository.GetAllInstalledApps +type InstalledAppsWithChartDetails struct { + AppStoreApplicationName string `json:"app_store_application_name"` + ChartRepoName string `json:"chart_repo_name"` + DockerArtifactStoreId string `json:"docker_artifact_store_id"` + AppName string `json:"app_name"` + EnvironmentName string `json:"environment_name"` + InstalledAppVersionId int `json:"installed_app_version_id"` + AppStoreApplicationVersionId int `json:"app_store_application_version_id"` + Icon string `json:"icon"` + Readme string `json:"readme"` + CreatedOn time.Time `json:"created_on"` + UpdatedOn time.Time `json:"updated_on"` + Id int `json:"id"` + EnvironmentId int `json:"environment_id"` + Deprecated bool `json:"deprecated"` + ClusterName string `json:"clusterName"` + Namespace string `json:"namespace"` + TeamId int `json:"teamId"` + ClusterId int `json:"clusterId"` + AppOfferingMode string `json:"app_offering_mode"` + AppStatus string `json:"app_status"` + DeploymentAppDeleteRequest bool `json:"deploymentAppDeleteRequest"` +} + +// InstalledAppAndEnvDetails is used to operate on native query; This should be avoided. +// Usage functions: InstalledAppRepository.GetAllInstalledAppsByChartRepoId and InstalledAppRepository.GetAllInstalledAppsByAppStoreId +type InstalledAppAndEnvDetails struct { + EnvironmentName string `json:"environment_name"` + EnvironmentId int `json:"environment_id"` + AppName string `json:"app_name"` + AppOfferingMode string `json:"appOfferingMode"` + UpdatedOn time.Time `json:"updated_on"` + EmailId string `json:"email_id"` + InstalledAppVersionId int `json:"installed_app_version_id"` + AppId int `json:"app_id"` + InstalledAppId int `json:"installed_app_id"` + AppStoreApplicationVersionId int `json:"app_store_application_version_id"` + AppStatus string `json:"app_status"` + DeploymentAppType string `json:"-"` +} + +// InstallAppDeleteRequest is used to operate on native query; This should be avoided. +// Usage func: InstalledAppRepository.GetInstalledAppByGitHash +type InstallAppDeleteRequest struct { + InstalledAppId int `json:"installed_app_id,omitempty,notnull"` + AppName string `json:"app_name,omitempty"` + AppId int `json:"app_id,omitempty"` + EnvironmentId int `json:"environment_id,omitempty"` + AppOfferingMode string `json:"app_offering_mode"` + ClusterId int `json:"cluster_id"` + Namespace string `json:"namespace"` +} diff --git a/pkg/appStore/deployment/repository/InstalledAppRepository.go b/pkg/appStore/deployment/repository/InstalledAppRepository.go index 134887be61..d4d91de2a1 100644 --- a/pkg/appStore/deployment/repository/InstalledAppRepository.go +++ b/pkg/appStore/deployment/repository/InstalledAppRepository.go @@ -31,9 +31,62 @@ import ( "github.com/go-pg/pg/orm" "go.uber.org/zap" "strconv" - "time" ) +type InstalledApps struct { + TableName struct{} `sql:"installed_apps" pg:",discard_unknown_columns"` + Id int `sql:"id,pk"` + AppId int `sql:"app_id,notnull"` + EnvironmentId int `sql:"environment_id,notnull"` + Active bool `sql:"active, notnull"` + GitOpsRepoName string `sql:"git_ops_repo_name"` + DeploymentAppType string `sql:"deployment_app_type"` // TODO: instead of string, use enum + Status appStoreBean.AppstoreDeploymentStatus `sql:"status"` + DeploymentAppDeleteRequest bool `sql:"deployment_app_delete_request"` + Notes string `json:"notes"` + App app.App + Environment repository.Environment + sql.AuditLog +} + +func (model *InstalledApps) MarkActive() { + model.Active = true +} + +func (model *InstalledApps) MarkInActive() { + model.Active = false +} + +func (model *InstalledApps) UpdateStatus(status appStoreBean.AppstoreDeploymentStatus) { + model.Status = status +} + +func (model *InstalledApps) UpdateGitOpsRepoName(gitOpsRepoName string) { + model.GitOpsRepoName = gitOpsRepoName +} + +type InstalledAppVersions struct { + TableName struct{} `sql:"installed_app_versions" pg:",discard_unknown_columns"` + Id int `sql:"id,pk"` + InstalledAppId int `sql:"installed_app_id,notnull"` + AppStoreApplicationVersionId int `sql:"app_store_application_version_id,notnull"` + ValuesYaml string `sql:"values_yaml_raw"` + Active bool `sql:"active, notnull"` + ReferenceValueId int `sql:"reference_value_id"` + ReferenceValueKind string `sql:"reference_value_kind"` + sql.AuditLog + InstalledApp InstalledApps + AppStoreApplicationVersion appStoreDiscoverRepository.AppStoreApplicationVersion +} + +func (model *InstalledAppVersions) MarkActive() { + model.Active = true +} + +func (model *InstalledAppVersions) MarkInActive() { + model.Active = false +} + type InstalledAppRepository interface { CreateInstalledApp(model *InstalledApps, tx *pg.Tx) (*InstalledApps, error) CreateInstalledAppVersion(model *InstalledAppVersions, tx *pg.Tx) (*InstalledAppVersions, error) @@ -43,7 +96,7 @@ type InstalledAppRepository interface { GetInstalledAppVersion(id int) (*InstalledAppVersions, error) GetInstalledAppVersionAny(id int) (*InstalledAppVersions, error) GetAllInstalledApps(filter *appStoreBean.AppStoreFilter) ([]InstalledAppsWithChartDetails, error) - GetAllIntalledAppsByAppStoreId(appStoreId int) ([]InstalledAppAndEnvDetails, error) + GetAllInstalledAppsByAppStoreId(appStoreId int) ([]InstalledAppAndEnvDetails, error) GetAllInstalledAppsByChartRepoId(chartRepoId int) ([]InstalledAppAndEnvDetails, error) GetInstalledAppVersionByInstalledAppIdAndEnvId(installedAppId int, envId int) (*InstalledAppVersions, error) FetchNotes(installedAppId int) (*InstalledApps, error) @@ -81,94 +134,10 @@ type InstalledAppRepositoryImpl struct { Logger *zap.SugaredLogger } -type InstallAppDeleteRequest struct { - InstalledAppId int `json:"installed_app_id,omitempty,notnull"` - AppName string `json:"app_name,omitempty"` - AppId int `json:"app_id,omitempty"` - EnvironmentId int `json:"environment_id,omitempty"` - AppOfferingMode string `json:"app_offering_mode"` - ClusterId int `json:"cluster_id"` - Namespace string `json:"namespace"` -} - func NewInstalledAppRepositoryImpl(Logger *zap.SugaredLogger, dbConnection *pg.DB) *InstalledAppRepositoryImpl { return &InstalledAppRepositoryImpl{dbConnection: dbConnection, Logger: Logger} } -type InstalledApps struct { - TableName struct{} `sql:"installed_apps" pg:",discard_unknown_columns"` - Id int `sql:"id,pk"` - AppId int `sql:"app_id,notnull"` - EnvironmentId int `sql:"environment_id,notnull"` - Active bool `sql:"active, notnull"` - GitOpsRepoName string `sql:"git_ops_repo_name"` - DeploymentAppType string `sql:"deployment_app_type"` - Status appStoreBean.AppstoreDeploymentStatus `sql:"status"` - DeploymentAppDeleteRequest bool `sql:"deployment_app_delete_request"` - Notes string `json:"notes"` - App app.App - Environment repository.Environment - sql.AuditLog -} - -type InstalledAppVersions struct { - TableName struct{} `sql:"installed_app_versions" pg:",discard_unknown_columns"` - Id int `sql:"id,pk"` - InstalledAppId int `sql:"installed_app_id,notnull"` - AppStoreApplicationVersionId int `sql:"app_store_application_version_id,notnull"` - ValuesYaml string `sql:"values_yaml_raw"` - Active bool `sql:"active, notnull"` - ReferenceValueId int `sql:"reference_value_id"` - ReferenceValueKind string `sql:"reference_value_kind"` - sql.AuditLog - InstalledApp InstalledApps - AppStoreApplicationVersion appStoreDiscoverRepository.AppStoreApplicationVersion -} - -type GitOpsAppDetails struct { - GitOpsAppName string `sql:"git_ops_app_name"` - InstalledAppId int `sql:"installed_app_id"` -} - -type InstalledAppsWithChartDetails struct { - AppStoreApplicationName string `json:"app_store_application_name"` - ChartRepoName string `json:"chart_repo_name"` - DockerArtifactStoreId string `json:"docker_artifact_store_id"` - AppName string `json:"app_name"` - EnvironmentName string `json:"environment_name"` - InstalledAppVersionId int `json:"installed_app_version_id"` - AppStoreApplicationVersionId int `json:"app_store_application_version_id"` - Icon string `json:"icon"` - Readme string `json:"readme"` - CreatedOn time.Time `json:"created_on"` - UpdatedOn time.Time `json:"updated_on"` - Id int `json:"id"` - EnvironmentId int `json:"environment_id"` - Deprecated bool `json:"deprecated"` - ClusterName string `json:"clusterName"` - Namespace string `json:"namespace"` - TeamId int `json:"teamId"` - ClusterId int `json:"clusterId"` - AppOfferingMode string `json:"app_offering_mode"` - AppStatus string `json:"app_status"` - DeploymentAppDeleteRequest bool `json:"deploymentAppDeleteRequest"` -} - -type InstalledAppAndEnvDetails struct { - EnvironmentName string `json:"environment_name"` - EnvironmentId int `json:"environment_id"` - AppName string `json:"app_name"` - AppOfferingMode string `json:"appOfferingMode"` - UpdatedOn time.Time `json:"updated_on"` - EmailId string `json:"email_id"` - InstalledAppVersionId int `json:"installed_app_version_id"` - AppId int `json:"app_id"` - InstalledAppId int `json:"installed_app_id"` - AppStoreApplicationVersionId int `json:"app_store_application_version_id"` - AppStatus string `json:"app_status"` - DeploymentAppType string `json:"-"` -} - func (impl InstalledAppRepositoryImpl) CreateInstalledApp(model *InstalledApps, tx *pg.Tx) (*InstalledApps, error) { err := tx.Insert(model) if err != nil { @@ -427,7 +396,7 @@ func (impl InstalledAppRepositoryImpl) GetAllInstalledApps(filter *appStoreBean. return installedAppsWithChartDetails, err } -func (impl InstalledAppRepositoryImpl) GetAllIntalledAppsByAppStoreId(appStoreId int) ([]InstalledAppAndEnvDetails, error) { +func (impl InstalledAppRepositoryImpl) GetAllInstalledAppsByAppStoreId(appStoreId int) ([]InstalledAppAndEnvDetails, error) { var installedAppAndEnvDetails []InstalledAppAndEnvDetails var queryTemp = "select env.environment_name, env.id as environment_id, a.app_name, a.app_offering_mode, ia.updated_on, u.email_id," + " asav.id as app_store_application_version_id, iav.id as installed_app_version_id, ia.id as installed_app_id, ia.app_id, ia.deployment_app_type, app_status.status as app_status" + diff --git a/pkg/appStore/deployment/repository/InstalledAppVersionHistory.go b/pkg/appStore/deployment/repository/InstalledAppVersionHistory.go index f914c749ac..4b2b127632 100644 --- a/pkg/appStore/deployment/repository/InstalledAppVersionHistory.go +++ b/pkg/appStore/deployment/repository/InstalledAppVersionHistory.go @@ -46,6 +46,18 @@ type InstalledAppVersionHistory struct { sql.AuditLog } +func (model *InstalledAppVersionHistory) SetStartedOn() { + model.StartedOn = time.Now() +} + +func (model *InstalledAppVersionHistory) SetFinishedOn() { + model.FinishedOn = time.Now() +} + +func (model *InstalledAppVersionHistory) SetStatus(status string) { + model.Status = status +} + func (impl InstalledAppVersionHistoryRepositoryImpl) CreateInstalledAppVersionHistory(model *InstalledAppVersionHistory, tx *pg.Tx) (*InstalledAppVersionHistory, error) { err := tx.Insert(model) if err != nil { diff --git a/pkg/appStore/deployment/service/AppStoreDeploymentDBService.go b/pkg/appStore/deployment/service/AppStoreDeploymentDBService.go new file mode 100644 index 0000000000..106684e385 --- /dev/null +++ b/pkg/appStore/deployment/service/AppStoreDeploymentDBService.go @@ -0,0 +1,180 @@ +package service + +import ( + "fmt" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/appStore/adapter" + appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" + "github.com/devtron-labs/devtron/pkg/bean" + util2 "github.com/devtron-labs/devtron/util" + "github.com/go-pg/pg" + "net/http" +) + +func (impl AppStoreDeploymentServiceImpl) AppStoreDeployOperationDB(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, tx *pg.Tx, skipAppCreation bool) (*appStoreBean.InstallAppVersionDTO, error) { + + var isInternalUse = impl.deploymentTypeConfig.IsInternalUse + + isGitOpsConfigured := false + gitOpsConfig, err := impl.gitOpsRepository.GetGitOpsConfigActive() + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("GetGitOpsConfigActive, error while getting", "err", err) + return nil, err + } + if gitOpsConfig != nil && gitOpsConfig.Id > 0 { + isGitOpsConfigured = true + } + + if isInternalUse && !isGitOpsConfigured && installAppVersionRequest.DeploymentAppType == util.PIPELINE_DEPLOYMENT_TYPE_ACD { + impl.logger.Errorw("gitops not configured but selected for CD") + err := &util.ApiError{ + HttpStatusCode: http.StatusBadRequest, + InternalMessage: "Gitops integration is not installed/configured. Please install/configure gitops or use helm option.", + UserMessage: "Gitops integration is not installed/configured. Please install/configure gitops or use helm option.", + } + return nil, err + } + + appStoreAppVersion, err := impl.appStoreApplicationVersionRepository.FindById(installAppVersionRequest.AppStoreVersion) + if err != nil { + impl.logger.Errorw("fetching error", "err", err) + return nil, err + } + + var isOCIRepo bool + if appStoreAppVersion.AppStore.DockerArtifactStore != nil { + isOCIRepo = true + } else { + isOCIRepo = false + } + + var appInstallationMode string + if util2.IsBaseStack() || util2.IsHelmApp(installAppVersionRequest.AppOfferingMode) { + appInstallationMode = util2.SERVER_MODE_HYPERION + } else { + appInstallationMode = util2.SERVER_MODE_FULL + } + + // create env if env not exists for clusterId and namespace for hyperion mode + if util2.IsHelmApp(appInstallationMode) { + envId, err := impl.createEnvironmentIfNotExists(installAppVersionRequest) + if err != nil { + return nil, err + } + installAppVersionRequest.EnvironmentId = envId + } + + environment, err := impl.environmentRepository.FindById(installAppVersionRequest.EnvironmentId) + if err != nil { + impl.logger.Errorw("fetching error", "err", err) + return nil, err + } + installAppVersionRequest.Environment = environment + installAppVersionRequest.ACDAppName = fmt.Sprintf("%s-%s", installAppVersionRequest.AppName, installAppVersionRequest.Environment.Name) + installAppVersionRequest.ClusterId = environment.ClusterId + appCreateRequest := &bean.CreateAppDTO{ + Id: installAppVersionRequest.AppId, + AppName: installAppVersionRequest.AppName, + TeamId: installAppVersionRequest.TeamId, + UserId: installAppVersionRequest.UserId, + } + + appCreateRequest, err = impl.createAppForAppStore(appCreateRequest, tx, appInstallationMode, skipAppCreation) + if err != nil { + impl.logger.Errorw("error while creating app", "error", err) + return nil, err + } + installAppVersionRequest.AppId = appCreateRequest.Id + + if !isInternalUse { + if isGitOpsConfigured && appInstallationMode == util2.SERVER_MODE_FULL && !isOCIRepo { + installAppVersionRequest.DeploymentAppType = util.PIPELINE_DEPLOYMENT_TYPE_ACD + } else { + installAppVersionRequest.DeploymentAppType = util.PIPELINE_DEPLOYMENT_TYPE_HELM + } + } + if installAppVersionRequest.DeploymentAppType == "" { + if isGitOpsConfigured && !isOCIRepo { + installAppVersionRequest.DeploymentAppType = util.PIPELINE_DEPLOYMENT_TYPE_ACD + } else { + installAppVersionRequest.DeploymentAppType = util.PIPELINE_DEPLOYMENT_TYPE_HELM + } + } + + if util2.IsFullStack() { + installAppVersionRequest.GitOpsRepoName = impl.ChartTemplateService.GetGitOpsRepoName(installAppVersionRequest.AppName) + } + + installedAppModel := adapter.NewInstallAppModel(installAppVersionRequest, appStoreBean.DEPLOY_INIT) + installedApp, err := impl.installedAppRepository.CreateInstalledApp(installedAppModel, tx) + if err != nil { + impl.logger.Errorw("error while creating install app", "error", err) + return nil, err + } + installAppVersionRequest.InstalledAppId = installedApp.Id + + installedAppVersions := adapter.NewInstallAppVersionsModel(installAppVersionRequest) + _, err = impl.installedAppRepository.CreateInstalledAppVersion(installedAppVersions, tx) + if err != nil { + impl.logger.Errorw("error while fetching from db", "error", err) + return nil, err + } + + installAppVersionRequest.InstalledAppVersionId = installedAppVersions.Id + installAppVersionRequest.Id = installedAppVersions.Id + + helmInstallConfigDTO := appStoreBean.HelmReleaseStatusConfig{ + InstallAppVersionHistoryId: 0, + Message: "Install initiated", + IsReleaseInstalled: false, + ErrorInInstallation: false, + } + installedAppVersionHistory, err := adapter.NewInstallAppVersionHistoryModel(installAppVersionRequest, pipelineConfig.WorkflowInProgress, helmInstallConfigDTO) + if err != nil { + impl.logger.Errorw("error in helm install config marshal", "err", err) + } + _, err = impl.installedAppRepositoryHistory.CreateInstalledAppVersionHistory(installedAppVersionHistory, tx) + if err != nil { + impl.logger.Errorw("error while fetching from db", "error", err) + return nil, err + } + + installAppVersionRequest.InstalledAppVersionHistoryId = installedAppVersionHistory.Id + if installAppVersionRequest.DefaultClusterComponent { + clusterInstalledAppsModel := adapter.NewClusterInstalledAppsModel(installAppVersionRequest, environment.ClusterId) + err = impl.clusterInstalledAppsRepository.Save(clusterInstalledAppsModel, tx) + if err != nil { + impl.logger.Errorw("error while creating cluster install app", "error", err) + return nil, err + } + } + return installAppVersionRequest, nil +} + +func (impl AppStoreDeploymentServiceImpl) AppStoreDeployOperationStatusUpdate(installAppId int, status appStoreBean.AppstoreDeploymentStatus) (bool, error) { + dbConnection := impl.installedAppRepository.GetConnection() + tx, err := dbConnection.Begin() + if err != nil { + return false, err + } + // Rollback tx on error. + defer tx.Rollback() + installedApp, err := impl.installedAppRepository.GetInstalledApp(installAppId) + if err != nil { + impl.logger.Errorw("error while fetching from db", "error", err) + return false, err + } + installedApp.Status = status + _, err = impl.installedAppRepository.UpdateInstalledApp(installedApp, tx) + if err != nil { + impl.logger.Errorw("error while fetching from db", "error", err) + return false, err + } + err = tx.Commit() + if err != nil { + impl.logger.Errorw("error while commit db transaction to db", "error", err) + return false, err + } + return true, nil +} diff --git a/pkg/appStore/deployment/service/AppStoreDeploymentService.go b/pkg/appStore/deployment/service/AppStoreDeploymentService.go index a06b2c76dc..597eb63971 100644 --- a/pkg/appStore/deployment/service/AppStoreDeploymentService.go +++ b/pkg/appStore/deployment/service/AppStoreDeploymentService.go @@ -33,6 +33,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/helper" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/appStore/adapter" appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" repository3 "github.com/devtron-labs/devtron/pkg/appStore/chartGroup/repository" appStoreDeploymentCommon "github.com/devtron-labs/devtron/pkg/appStore/deployment/common" @@ -72,7 +73,6 @@ type AppStoreDeploymentService interface { GetInstalledAppVersion(id int, userId int32) (*appStoreBean.InstallAppVersionDTO, error) InstallAppByHelm(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, ctx context.Context) (*appStoreBean.InstallAppVersionDTO, error) UpdateProjectHelmApp(updateAppRequest *appStoreBean.UpdateProjectHelmAppDTO) error - UpdateNotesForInstalledApp(installAppId int, notes string) (bool, error) UpdatePreviousDeploymentStatusForAppStore(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, triggeredAt time.Time, err error) error UpdateInstallAppVersionHistoryStatus(installedAppVersionHistoryId int, status string) error } @@ -141,238 +141,6 @@ func NewAppStoreDeploymentServiceImpl(logger *zap.SugaredLogger, installedAppRep return appStoreDeploymentServiceImpl } -func (impl AppStoreDeploymentServiceImpl) AppStoreDeployOperationDB(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, tx *pg.Tx, skipAppCreation bool) (*appStoreBean.InstallAppVersionDTO, error) { - - var isInternalUse = impl.deploymentTypeConfig.IsInternalUse - - isGitOpsConfigured := false - gitOpsConfig, err := impl.gitOpsRepository.GetGitOpsConfigActive() - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("GetGitOpsConfigActive, error while getting", "err", err) - return nil, err - } - if gitOpsConfig != nil && gitOpsConfig.Id > 0 { - isGitOpsConfigured = true - } - - if isInternalUse && !isGitOpsConfigured && installAppVersionRequest.DeploymentAppType == util.PIPELINE_DEPLOYMENT_TYPE_ACD { - impl.logger.Errorw("gitops not configured but selected for CD") - err := &util.ApiError{ - HttpStatusCode: http.StatusBadRequest, - InternalMessage: "Gitops integration is not installed/configured. Please install/configure gitops or use helm option.", - UserMessage: "Gitops integration is not installed/configured. Please install/configure gitops or use helm option.", - } - return nil, err - } - - appStoreAppVersion, err := impl.appStoreApplicationVersionRepository.FindById(installAppVersionRequest.AppStoreVersion) - if err != nil { - impl.logger.Errorw("fetching error", "err", err) - return nil, err - } - - var isOCIRepo bool - if appStoreAppVersion.AppStore.DockerArtifactStore != nil { - isOCIRepo = true - } else { - isOCIRepo = false - } - - var appInstallationMode string - if util2.IsBaseStack() || util2.IsHelmApp(installAppVersionRequest.AppOfferingMode) { - appInstallationMode = util2.SERVER_MODE_HYPERION - } else { - appInstallationMode = util2.SERVER_MODE_FULL - } - - // create env if env not exists for clusterId and namespace for hyperion mode - if util2.IsHelmApp(appInstallationMode) { - envId, err := impl.createEnvironmentIfNotExists(installAppVersionRequest) - if err != nil { - return nil, err - } - installAppVersionRequest.EnvironmentId = envId - } - - environment, err := impl.environmentRepository.FindById(installAppVersionRequest.EnvironmentId) - if err != nil { - impl.logger.Errorw("fetching error", "err", err) - return nil, err - } - installAppVersionRequest.Environment = environment - installAppVersionRequest.ACDAppName = fmt.Sprintf("%s-%s", installAppVersionRequest.AppName, installAppVersionRequest.Environment.Name) - installAppVersionRequest.ClusterId = environment.ClusterId - appCreateRequest := &bean.CreateAppDTO{ - Id: installAppVersionRequest.AppId, - AppName: installAppVersionRequest.AppName, - TeamId: installAppVersionRequest.TeamId, - UserId: installAppVersionRequest.UserId, - } - - appCreateRequest, err = impl.createAppForAppStore(appCreateRequest, tx, appInstallationMode, skipAppCreation) - if err != nil { - impl.logger.Errorw("error while creating app", "error", err) - return nil, err - } - installAppVersionRequest.AppId = appCreateRequest.Id - - installedAppModel := &repository.InstalledApps{ - AppId: appCreateRequest.Id, - EnvironmentId: environment.Id, - Status: appStoreBean.DEPLOY_INIT, - } - - if !isInternalUse { - if isGitOpsConfigured && appInstallationMode == util2.SERVER_MODE_FULL && !isOCIRepo { - installAppVersionRequest.DeploymentAppType = util.PIPELINE_DEPLOYMENT_TYPE_ACD - } else { - installAppVersionRequest.DeploymentAppType = util.PIPELINE_DEPLOYMENT_TYPE_HELM - } - } - if installAppVersionRequest.DeploymentAppType == "" { - if isGitOpsConfigured && !isOCIRepo { - installAppVersionRequest.DeploymentAppType = util.PIPELINE_DEPLOYMENT_TYPE_ACD - } else { - installAppVersionRequest.DeploymentAppType = util.PIPELINE_DEPLOYMENT_TYPE_HELM - } - } - - installedAppModel.DeploymentAppType = installAppVersionRequest.DeploymentAppType - installedAppModel.CreatedBy = installAppVersionRequest.UserId - installedAppModel.UpdatedBy = installAppVersionRequest.UserId - installedAppModel.CreatedOn = time.Now() - installedAppModel.UpdatedOn = time.Now() - installedAppModel.Active = true - if util2.IsFullStack() { - installedAppModel.GitOpsRepoName = impl.ChartTemplateService.GetGitOpsRepoName(installAppVersionRequest.AppName) - installAppVersionRequest.GitOpsRepoName = installedAppModel.GitOpsRepoName - } - installedApp, err := impl.installedAppRepository.CreateInstalledApp(installedAppModel, tx) - if err != nil { - impl.logger.Errorw("error while creating install app", "error", err) - return nil, err - } - installAppVersionRequest.InstalledAppId = installedApp.Id - - installedAppVersions := &repository.InstalledAppVersions{ - InstalledAppId: installAppVersionRequest.InstalledAppId, - AppStoreApplicationVersionId: appStoreAppVersion.Id, - ValuesYaml: installAppVersionRequest.ValuesOverrideYaml, - //Values: "{}", - } - installedAppVersions.CreatedBy = installAppVersionRequest.UserId - installedAppVersions.UpdatedBy = installAppVersionRequest.UserId - installedAppVersions.CreatedOn = time.Now() - installedAppVersions.UpdatedOn = time.Now() - installedAppVersions.Active = true - installedAppVersions.ReferenceValueId = installAppVersionRequest.ReferenceValueId - installedAppVersions.ReferenceValueKind = installAppVersionRequest.ReferenceValueKind - _, err = impl.installedAppRepository.CreateInstalledAppVersion(installedAppVersions, tx) - if err != nil { - impl.logger.Errorw("error while fetching from db", "error", err) - return nil, err - } - installAppVersionRequest.InstalledAppVersionId = installedAppVersions.Id - installAppVersionRequest.Id = installedAppVersions.Id - - installedAppVersionHistory := &repository.InstalledAppVersionHistory{} - installedAppVersionHistory.InstalledAppVersionId = installedAppVersions.Id - installedAppVersionHistory.ValuesYamlRaw = installAppVersionRequest.ValuesOverrideYaml - installedAppVersionHistory.CreatedBy = installAppVersionRequest.UserId - installedAppVersionHistory.CreatedOn = time.Now() - installedAppVersionHistory.UpdatedBy = installAppVersionRequest.UserId - installedAppVersionHistory.UpdatedOn = time.Now() - installedAppVersionHistory.StartedOn = time.Now() - installedAppVersionHistory.Status = pipelineConfig.WorkflowInProgress - helmInstallConfigDTO := appStoreBean.HelmReleaseStatusConfig{ - InstallAppVersionHistoryId: 0, - Message: "Install initiated", - IsReleaseInstalled: false, - ErrorInInstallation: false, - } - helmInstallConfig, err := json.Marshal(helmInstallConfigDTO) - if err != nil { - impl.logger.Errorw("error in helm install config marshal", "err") - } - installedAppVersionHistory.HelmReleaseStatusConfig = string(helmInstallConfig) - _, err = impl.installedAppRepositoryHistory.CreateInstalledAppVersionHistory(installedAppVersionHistory, tx) - if err != nil { - impl.logger.Errorw("error while fetching from db", "error", err) - return nil, err - } - installAppVersionRequest.InstalledAppVersionHistoryId = installedAppVersionHistory.Id - if installAppVersionRequest.DefaultClusterComponent { - clusterInstalledAppsModel := &repository.ClusterInstalledApps{ - ClusterId: environment.ClusterId, - InstalledAppId: installAppVersionRequest.InstalledAppId, - } - clusterInstalledAppsModel.CreatedBy = installAppVersionRequest.UserId - clusterInstalledAppsModel.UpdatedBy = installAppVersionRequest.UserId - clusterInstalledAppsModel.CreatedOn = time.Now() - clusterInstalledAppsModel.UpdatedOn = time.Now() - err = impl.clusterInstalledAppsRepository.Save(clusterInstalledAppsModel, tx) - if err != nil { - impl.logger.Errorw("error while creating cluster install app", "error", err) - return nil, err - } - } - return installAppVersionRequest, nil -} - -func (impl AppStoreDeploymentServiceImpl) UpdateNotesForInstalledApp(installAppId int, notes string) (bool, error) { - dbConnection := impl.installedAppRepository.GetConnection() - tx, err := dbConnection.Begin() - if err != nil { - return false, err - } - // Rollback tx on error. - defer tx.Rollback() - installedApp, err := impl.installedAppRepository.GetInstalledApp(installAppId) - if err != nil { - impl.logger.Errorw("error while fetching from db", "error", err) - return false, err - } - installedApp.Notes = notes - _, err = impl.installedAppRepository.UpdateInstalledApp(installedApp, tx) - if err != nil { - impl.logger.Errorw("error while fetching from db", "error", err) - return false, err - } - err = tx.Commit() - if err != nil { - impl.logger.Errorw("error while commit db transaction to db", "error", err) - return false, err - } - return true, nil -} - -func (impl AppStoreDeploymentServiceImpl) AppStoreDeployOperationStatusUpdate(installAppId int, status appStoreBean.AppstoreDeploymentStatus) (bool, error) { - dbConnection := impl.installedAppRepository.GetConnection() - tx, err := dbConnection.Begin() - if err != nil { - return false, err - } - // Rollback tx on error. - defer tx.Rollback() - installedApp, err := impl.installedAppRepository.GetInstalledApp(installAppId) - if err != nil { - impl.logger.Errorw("error while fetching from db", "error", err) - return false, err - } - installedApp.Status = status - _, err = impl.installedAppRepository.UpdateInstalledApp(installedApp, tx) - if err != nil { - impl.logger.Errorw("error while fetching from db", "error", err) - return false, err - } - err = tx.Commit() - if err != nil { - impl.logger.Errorw("error while commit db transaction to db", "error", err) - return false, err - } - return true, nil -} - func (impl *AppStoreDeploymentServiceImpl) IsChartRepoActive(appStoreVersionId int) (bool, error) { appStoreAppVersion, err := impl.appStoreApplicationVersionRepository.FindById(appStoreVersionId) if err != nil { @@ -402,7 +170,7 @@ func (impl AppStoreDeploymentServiceImpl) InstallApp(installAppVersionRequest *a impl.logger.Errorw(" error", "err", err) return nil, err } - impl.updateDeploymentParametersInRequest(installAppVersionRequest, installAppVersionRequest.DeploymentAppType) + installedAppDeploymentAction := adapter.NewInstalledAppDeploymentAction(installAppVersionRequest.DeploymentAppType) if util.IsAcdApp(installAppVersionRequest.DeploymentAppType) || util.IsManifestDownload(installAppVersionRequest.DeploymentAppType) { _ = impl.appStoreDeploymentArgoCdService.SaveTimelineForACDHelmApps(installAppVersionRequest, pipelineConfig.TIMELINE_STATUS_DEPLOYMENT_INITIATED, "Deployment initiated successfully.", time.Now(), tx) @@ -419,7 +187,7 @@ func (impl AppStoreDeploymentServiceImpl) InstallApp(installAppVersionRequest *a } var gitOpsResponse *appStoreDeploymentCommon.AppStoreGitOpsResponse - if installAppVersionRequest.PerformGitOps { + if installedAppDeploymentAction.PerformGitOps { gitOpsResponse, err = impl.appStoreDeploymentCommonService.GitOpsOperations(manifest, installAppVersionRequest) if err != nil { impl.logger.Errorw("error in doing gitops operation", "err", err) @@ -603,26 +371,12 @@ func (impl AppStoreDeploymentServiceImpl) GetInstalledApp(id int) (*appStoreBean impl.logger.Errorw("error while fetching from db", "error", err) return nil, err } - chartTemplate := impl.chartAdaptor2(app) + chartTemplate := adapter.GenerateInstallAppVersionMinDTO(app) return chartTemplate, nil } -// converts db object to bean -func (impl AppStoreDeploymentServiceImpl) chartAdaptor2(chart *repository.InstalledApps) *appStoreBean.InstallAppVersionDTO { - return &appStoreBean.InstallAppVersionDTO{ - EnvironmentId: chart.EnvironmentId, - InstalledAppId: chart.Id, - AppId: chart.AppId, - AppOfferingMode: chart.App.AppOfferingMode, - ClusterId: chart.Environment.ClusterId, - Namespace: chart.Environment.Namespace, - AppName: chart.App.AppName, - EnvironmentName: chart.Environment.Name, - } -} - func (impl AppStoreDeploymentServiceImpl) GetAllInstalledAppsByAppStoreId(w http.ResponseWriter, r *http.Request, token string, appStoreId int) ([]appStoreBean.InstalledAppsResponse, error) { - installedApps, err := impl.installedAppRepository.GetAllIntalledAppsByAppStoreId(appStoreId) + installedApps, err := impl.installedAppRepository.GetAllInstalledAppsByAppStoreId(appStoreId) if err != nil && !util.IsErrNoRows(err) { impl.logger.Error(err) return nil, err @@ -1298,27 +1052,6 @@ func (impl *AppStoreDeploymentServiceImpl) CheckIfMonoRepoMigrationRequired(inst return monoRepoMigrationRequired } -func (impl *AppStoreDeploymentServiceImpl) updateDeploymentParametersInRequest(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, deploymentAppType string) { - - installAppVersionRequest.DeploymentAppType = deploymentAppType - - switch installAppVersionRequest.DeploymentAppType { - case util.PIPELINE_DEPLOYMENT_TYPE_ACD: - installAppVersionRequest.PerformGitOps = true - installAppVersionRequest.PerformACDDeployment = true - installAppVersionRequest.PerformHelmDeployment = false - case util.PIPELINE_DEPLOYMENT_TYPE_HELM: - installAppVersionRequest.PerformGitOps = false - installAppVersionRequest.PerformACDDeployment = false - installAppVersionRequest.PerformHelmDeployment = true - case util.PIPELINE_DEPLOYMENT_TYPE_MANIFEST_DOWNLOAD: - installAppVersionRequest.PerformGitOps = false - installAppVersionRequest.PerformHelmDeployment = false - installAppVersionRequest.PerformACDDeployment = false - } - -} - func (impl *AppStoreDeploymentServiceImpl) UpdateInstalledApp(ctx context.Context, installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*appStoreBean.InstallAppVersionDTO, error) { // db operations @@ -1334,8 +1067,9 @@ func (impl *AppStoreDeploymentServiceImpl) UpdateInstalledApp(ctx context.Contex if err != nil { return nil, err } + installAppVersionRequest.UpdateDeploymentAppType(installedApp.DeploymentAppType) - impl.updateDeploymentParametersInRequest(installAppVersionRequest, installedApp.DeploymentAppType) + installedAppDeploymentAction := adapter.NewInstalledAppDeploymentAction(installedApp.DeploymentAppType) var installedAppVersion *repository.InstalledAppVersions @@ -1433,7 +1167,7 @@ func (impl *AppStoreDeploymentServiceImpl) UpdateInstalledApp(ctx context.Contex monoRepoMigrationRequired := false gitOpsResponse := &appStoreDeploymentCommon.AppStoreGitOpsResponse{} - if installAppVersionRequest.PerformGitOps { + if installedAppDeploymentAction.PerformGitOps { // required if gitOps repo name is changed, gitOps repo name will change if env variable which we use as suffix changes gitOpsRepoName := installedApp.GitOpsRepoName @@ -1511,14 +1245,14 @@ func (impl *AppStoreDeploymentServiceImpl) UpdateInstalledApp(ctx context.Contex } } - if installAppVersionRequest.PerformACDDeployment { + if installedAppDeploymentAction.PerformACDDeployment { // refresh update repo details on ArgoCD if repo is changed err = impl.appStoreDeploymentArgoCdService.UpdateAndSyncACDApps(installAppVersionRequest, gitOpsResponse.ChartGitAttribute, monoRepoMigrationRequired, ctx, tx) if err != nil { impl.logger.Errorw("error in acd patch request", "err", err) return nil, err } - } else if installAppVersionRequest.PerformHelmDeployment { + } else if installedAppDeploymentAction.PerformHelmDeployment { err = impl.appStoreDeploymentHelmService.UpdateChartInfo(installAppVersionRequest, gitOpsResponse.ChartGitAttribute, installAppVersionRequest.InstalledAppVersionHistoryId, ctx) if err != nil { if err != nil { diff --git a/pkg/appStore/deployment/service/AppStoreDeploymentService_test.go b/pkg/appStore/deployment/service/AppStoreDeploymentService_test.go index 79a6fd1f9f..e0589afb1b 100644 --- a/pkg/appStore/deployment/service/AppStoreDeploymentService_test.go +++ b/pkg/appStore/deployment/service/AppStoreDeploymentService_test.go @@ -9,7 +9,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/util" appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" - repository5 "github.com/devtron-labs/devtron/pkg/appStore/chartGroup/repository" + repository6 "github.com/devtron-labs/devtron/pkg/appStore/chartGroup/repository" repository3 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" appStoreDiscoverRepository "github.com/devtron-labs/devtron/pkg/appStore/discover/repository" repository5 "github.com/devtron-labs/devtron/pkg/auth/user/repository" @@ -134,7 +134,7 @@ func initAppStoreDeploymentService(t *testing.T, internalUse bool) *AppStoreDepl db, _ := sql.NewDbConnection(config, sugaredLogger) gitOpsRepository := repository.NewGitOpsConfigRepositoryImpl(sugaredLogger, db) - chartGroupDeploymentRepository := repository5.NewChartGroupDeploymentRepositoryImpl(db, sugaredLogger) + chartGroupDeploymentRepository := repository6.NewChartGroupDeploymentRepositoryImpl(db, sugaredLogger) appStoreDiscoverRepository := appStoreDiscoverRepository.NewAppStoreApplicationVersionRepositoryImpl(sugaredLogger, db) @@ -171,11 +171,11 @@ func initAppStoreDeploymentService(t *testing.T, internalUse bool) *AppStoreDepl clusterService, nil, nil, - nil, InstalledAppVersionHistoryRepository, gitOpsRepository, nil, &DeploymentServiceTypeConfig{IsInternalUse: internalUse}, + nil, nil) return AppStoreDeploymentServiceImpl diff --git a/pkg/appStore/deployment/service/InstalledAppService.go b/pkg/appStore/deployment/service/InstalledAppService.go index aebe347bcd..a6192db3e6 100644 --- a/pkg/appStore/deployment/service/InstalledAppService.go +++ b/pkg/appStore/deployment/service/InstalledAppService.go @@ -172,7 +172,7 @@ func NewInstalledAppServiceImpl(logger *zap.SugaredLogger, k8sApplicationService: k8sApplicationService, acdConfig: acdConfig, } - err := impl.Subscribe() + err := impl.subscribe() if err != nil { return nil, err } @@ -606,7 +606,7 @@ func (impl *InstalledAppServiceImpl) triggerDeploymentEvent(installAppVersions [ } } -func (impl *InstalledAppServiceImpl) Subscribe() error { +func (impl *InstalledAppServiceImpl) subscribe() error { callback := func(msg *model.PubSubMsg) { impl.logger.Debug("cd stage event received") //defer msg.Ack() diff --git a/pkg/appStore/deployment/service/InstalledAppService_test.go b/pkg/appStore/deployment/service/InstalledAppService_test.go index b28d642ec6..4c70d39142 100644 --- a/pkg/appStore/deployment/service/InstalledAppService_test.go +++ b/pkg/appStore/deployment/service/InstalledAppService_test.go @@ -4,13 +4,10 @@ import ( "testing" pubsub "github.com/devtron-labs/common-lib/pubsub-lib" - "github.com/devtron-labs/devtron/client/argocdServer" "github.com/devtron-labs/devtron/client/argocdServer/application" - repository2 "github.com/devtron-labs/devtron/client/argocdServer/repository" repository3 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/util" - appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" repository5 "github.com/devtron-labs/devtron/pkg/appStore/chartGroup/repository" appStoreDeploymentFullMode "github.com/devtron-labs/devtron/pkg/appStore/deployment/fullMode" repository4 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" @@ -28,9 +25,6 @@ func TestInstalledAppServiceImpl_DeployDefaultChartOnCluster(t *testing.T) { type fields struct { logger *zap.SugaredLogger installedAppRepository repository4.InstalledAppRepository - chartTemplateService util.ChartTemplateService - refChartDir appStoreBean.RefChartProxyDir - repositoryService repository2.ServiceClient appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository environmentRepository repository.EnvironmentRepository teamRepository team.TeamRepository @@ -38,10 +32,8 @@ func TestInstalledAppServiceImpl_DeployDefaultChartOnCluster(t *testing.T) { acdClient application.ServiceClient appStoreValuesService service.AppStoreValuesService pubsubClient *pubsub.PubSubClientServiceImpl - tokenCache *util2.TokenCache chartGroupDeploymentRepository repository5.ChartGroupDeploymentRepository envService cluster.EnvironmentService - ArgoK8sClient argocdServer.ArgoK8sClient gitFactory *util.GitFactory aCDAuthConfig *util2.ACDAuthConfig gitOpsRepository repository3.GitOpsConfigRepository @@ -65,11 +57,9 @@ func TestInstalledAppServiceImpl_DeployDefaultChartOnCluster(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { impl := &InstalledAppServiceImpl{ - logger: tt.fields.logger, - installedAppRepository: tt.fields.installedAppRepository, - chartTemplateService: tt.fields.chartTemplateService, - refChartDir: tt.fields.refChartDir, - repositoryService: tt.fields.repositoryService, + logger: tt.fields.logger, + installedAppRepository: tt.fields.installedAppRepository, + appStoreApplicationVersionRepository: tt.fields.appStoreApplicationVersionRepository, environmentRepository: tt.fields.environmentRepository, teamRepository: tt.fields.teamRepository, @@ -77,10 +67,8 @@ func TestInstalledAppServiceImpl_DeployDefaultChartOnCluster(t *testing.T) { acdClient: tt.fields.acdClient, appStoreValuesService: tt.fields.appStoreValuesService, pubsubClient: tt.fields.pubsubClient, - tokenCache: tt.fields.tokenCache, chartGroupDeploymentRepository: tt.fields.chartGroupDeploymentRepository, envService: tt.fields.envService, - ArgoK8sClient: tt.fields.ArgoK8sClient, gitFactory: tt.fields.gitFactory, aCDAuthConfig: tt.fields.aCDAuthConfig, gitOpsRepository: tt.fields.gitOpsRepository, diff --git a/pkg/appStore/deployment/service/Notes.go b/pkg/appStore/deployment/service/Notes.go index 88ce454af6..33595afc25 100644 --- a/pkg/appStore/deployment/service/Notes.go +++ b/pkg/appStore/deployment/service/Notes.go @@ -9,6 +9,54 @@ import ( "regexp" ) +func (impl *InstalledAppServiceImpl) FetchChartNotes(installedAppId int, envId int, token string, checkNotesAuth func(token string, appName string, envId int) bool) (string, error) { + //check notes.txt in db + installedApp, err := impl.installedAppRepository.FetchNotes(installedAppId) + if err != nil && err != pg.ErrNoRows { + return "", err + } + installedAppVerison, err := impl.installedAppRepository.GetInstalledAppVersionByInstalledAppIdAndEnvId(installedAppId, envId) + if err != nil { + if err == pg.ErrNoRows { + return "", fmt.Errorf("values are outdated. please fetch the latest version and try again") + } + impl.logger.Errorw("error fetching installed app version in installed app service", "err", err) + return "", err + } + chartVersion := installedAppVerison.AppStoreApplicationVersion.Version + if err != nil { + impl.logger.Errorw("error fetching chart version in installed app service", "err", err) + return "", err + } + re := regexp.MustCompile(`CHART VERSION: ([0-9]+\.[0-9]+\.[0-9]+)`) + newStr := re.ReplaceAllString(installedApp.Notes, "CHART VERSION: "+chartVersion) + installedApp.Notes = newStr + appName := installedApp.App.AppName + if err != nil { + impl.logger.Errorw("error fetching notes from db", "err", err) + return "", err + } + isValidAuth := checkNotesAuth(token, appName, envId) + if !isValidAuth { + impl.logger.Errorw("unauthorized user", "isValidAuth", isValidAuth) + return "", fmt.Errorf("unauthorized user") + } + //if notes is not present in db then below call will happen + if installedApp.Notes == "" { + notes, _, err := impl.findNotesForArgoApplication(installedAppId, envId) + if err != nil { + impl.logger.Errorw("error fetching notes", "err", err) + return "", err + } + if notes == "" { + impl.logger.Errorw("error fetching notes", "err", err) + } + return notes, err + } + + return installedApp.Notes, nil +} + func (impl *InstalledAppServiceImpl) findNotesForArgoApplication(installedAppId, envId int) (string, string, error) { installedAppVerison, err := impl.installedAppRepository.GetInstalledAppVersionByInstalledAppIdAndEnvId(installedAppId, envId) if err != nil { @@ -60,7 +108,7 @@ func (impl *InstalledAppServiceImpl) findNotesForArgoApplication(installedAppId, impl.logger.Errorw("error in fetching notes", "err", err) return notes, appName, err } - _, err = impl.appStoreDeploymentService.UpdateNotesForInstalledApp(installedAppId, notes) + _, err = impl.updateNotesForInstalledApp(installedAppId, notes) if err != nil { impl.logger.Errorw("error in updating notes in db ", "err", err) return notes, appName, err @@ -69,50 +117,31 @@ func (impl *InstalledAppServiceImpl) findNotesForArgoApplication(installedAppId, return notes, appName, nil } -func (impl *InstalledAppServiceImpl) FetchChartNotes(installedAppId int, envId int, token string, checkNotesAuth func(token string, appName string, envId int) bool) (string, error) { - //check notes.txt in db - installedApp, err := impl.installedAppRepository.FetchNotes(installedAppId) - if err != nil && err != pg.ErrNoRows { - return "", err - } - installedAppVerison, err := impl.installedAppRepository.GetInstalledAppVersionByInstalledAppIdAndEnvId(installedAppId, envId) + +// updateNotesForInstalledApp will update the notes in repository.InstalledApps table +func (impl *InstalledAppServiceImpl) updateNotesForInstalledApp(installAppId int, notes string) (bool, error) { + dbConnection := impl.installedAppRepository.GetConnection() + tx, err := dbConnection.Begin() if err != nil { - if err == pg.ErrNoRows { - return "", fmt.Errorf("values are outdated. please fetch the latest version and try again") - } - impl.logger.Errorw("error fetching installed app version in installed app service", "err", err) - return "", err + return false, err } - chartVersion := installedAppVerison.AppStoreApplicationVersion.Version + // Rollback tx on error. + defer tx.Rollback() + installedApp, err := impl.installedAppRepository.GetInstalledApp(installAppId) if err != nil { - impl.logger.Errorw("error fetching chart version in installed app service", "err", err) - return "", err + impl.logger.Errorw("error while fetching from db", "error", err) + return false, err } - re := regexp.MustCompile(`CHART VERSION: ([0-9]+\.[0-9]+\.[0-9]+)`) - newStr := re.ReplaceAllString(installedApp.Notes, "CHART VERSION: "+chartVersion) - installedApp.Notes = newStr - appName := installedApp.App.AppName + installedApp.Notes = notes + _, err = impl.installedAppRepository.UpdateInstalledApp(installedApp, tx) if err != nil { - impl.logger.Errorw("error fetching notes from db", "err", err) - return "", err + impl.logger.Errorw("error while fetching from db", "error", err) + return false, err } - isValidAuth := checkNotesAuth(token, appName, envId) - if !isValidAuth { - impl.logger.Errorw("unauthorized user", "isValidAuth", isValidAuth) - return "", fmt.Errorf("unauthorized user") - } - //if notes is not present in db then below call will happen - if installedApp.Notes == "" { - notes, _, err := impl.findNotesForArgoApplication(installedAppId, envId) - if err != nil { - impl.logger.Errorw("error fetching notes", "err", err) - return "", err - } - if notes == "" { - impl.logger.Errorw("error fetching notes", "err", err) - } - return notes, err + err = tx.Commit() + if err != nil { + impl.logger.Errorw("error while commit db transaction to db", "error", err) + return false, err } - - return installedApp.Notes, nil + return true, nil } diff --git a/pkg/sql/UtilStructs.go b/pkg/sql/UtilStructs.go index bb8553f1e5..a067135431 100644 --- a/pkg/sql/UtilStructs.go +++ b/pkg/sql/UtilStructs.go @@ -36,3 +36,17 @@ func NewDefaultAuditLog(userId int32) AuditLog { UpdatedBy: userId, } } + +// CreateAuditLog can be used by any repository to create AuditLog for insert operation +func (model *AuditLog) CreateAuditLog(userId int32) { + model.CreatedOn = time.Now() + model.UpdatedOn = time.Now() + model.CreatedBy = userId + model.UpdatedBy = userId +} + +// UpdateAuditLog can be used by any repository to update AuditLog for update operation +func (model *AuditLog) UpdateAuditLog(userId int32) { + model.UpdatedOn = time.Now() + model.UpdatedBy = userId +} From 4842b83b1a744b8a0115b155e1af85021587e8f7 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Thu, 18 Jan 2024 17:30:20 +0530 Subject: [PATCH 26/64] review changes --- api/deployment/DeploymentConfigRestHandler.go | 9 ++- internal/util/ChartService.go | 12 ++-- pkg/app/AppService.go | 5 +- pkg/chart/ChartService.go | 4 +- .../repository/ChartRefRepository.go | 10 +-- .../chartRef/ChartRefService.go | 64 +++++++++---------- .../deploymentTemplate/chartRef/bean/bean.go | 11 ++-- .../DeployementTemplateService.go | 2 +- pkg/pipeline/WorkflowDagExecutor.go | 3 +- 9 files changed, 57 insertions(+), 63 deletions(-) diff --git a/api/deployment/DeploymentConfigRestHandler.go b/api/deployment/DeploymentConfigRestHandler.go index 8e27a52a5d..ac877b17cc 100644 --- a/api/deployment/DeploymentConfigRestHandler.go +++ b/api/deployment/DeploymentConfigRestHandler.go @@ -17,7 +17,6 @@ import ( "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" "github.com/devtron-labs/devtron/pkg/auth/user" "github.com/devtron-labs/devtron/pkg/chart" - chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" "github.com/devtron-labs/devtron/pkg/sql" "github.com/gorilla/mux" "github.com/juju/errors" @@ -99,7 +98,7 @@ func (handler *DeploymentConfigRestHandlerImpl) CreateChartFromFile(w http.Respo return } - chartInfo, err := handler.chartRefService.ExtractChartIfMissing(fileBytes, chartRepoRepository.RefChartDirPath, "") + chartInfo, err := handler.chartRefService.ExtractChartIfMissing(fileBytes, bean.RefChartDirPath, "") if err != nil { if chartInfo != nil && chartInfo.TemporaryFolder != "" { @@ -108,7 +107,7 @@ func (handler *DeploymentConfigRestHandlerImpl) CreateChartFromFile(w http.Respo handler.Logger.Errorw("error in deleting temp dir ", "err", err1) } } - if err.Error() == bean.CHART_ALREADY_EXISTS_INTERNAL_ERROR || err.Error() == bean.CHART_NAME_RESERVED_INTERNAL_ERROR { + if err.Error() == bean.ChartAlreadyExistsInternalError || err.Error() == bean.ChartNameReservedInternalError { common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } @@ -175,7 +174,7 @@ func (handler *DeploymentConfigRestHandlerImpl) SaveChart(w http.ResponseWriter, return } - location := filepath.Join(chartRepoRepository.RefChartDirPath, request.FileId) + location := filepath.Join(bean.RefChartDirPath, request.FileId) if request.Action == "Save" { file, err := ioutil.ReadFile(filepath.Join(location, "output.json")) if err != nil { @@ -231,7 +230,7 @@ func (handler *DeploymentConfigRestHandlerImpl) DownloadChart(w http.ResponseWri common.WriteJsonResp(w, fmt.Errorf("error in parsing chartRefId : %s must be integer", chartRefId), nil, http.StatusBadRequest) return } - manifestByteArr, err := handler.chartRefService.GetCustomChartInBytes(chartRefId) + manifestByteArr, err := handler.chartRefService.GetChartInBytes(chartRefId) if err != nil { handler.Logger.Errorw("error in converting chart to bytes", "err", err) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) diff --git a/internal/util/ChartService.go b/internal/util/ChartService.go index 2deb276c30..d40c5ff39e 100644 --- a/internal/util/ChartService.go +++ b/internal/util/ChartService.go @@ -51,7 +51,7 @@ const ( PIPELINE_DEPLOYMENT_TYPE_ACD = "argo_cd" PIPELINE_DEPLOYMENT_TYPE_HELM = "helm" PIPELINE_DEPLOYMENT_TYPE_MANIFEST_DOWNLOAD = "manifest_download" - ChartWorkingDirPath = "/tmp/charts/" + CHART_WORKING_DIR_PATH = "/tmp/charts/" ) type ChartCreateRequest struct { @@ -148,7 +148,7 @@ func (impl ChartTemplateServiceImpl) GetChartVersion(location string) (string, e func (impl ChartTemplateServiceImpl) FetchValuesFromReferenceChart(chartMetaData *chart.Metadata, refChartLocation string, templateName string, userId int32, pipelineStrategyPath string) (*ChartValues, *ChartGitAttribute, error) { chartMetaData.ApiVersion = "v1" // ensure always v1 dir := impl.GetDir() - chartDir := filepath.Join(ChartWorkingDirPath, dir) + chartDir := filepath.Join(CHART_WORKING_DIR_PATH, dir) impl.logger.Debugw("chart dir ", "chart", chartMetaData.Name, "dir", chartDir) err := os.MkdirAll(chartDir, os.ModePerm) //hack for concurrency handling if err != nil { @@ -188,7 +188,7 @@ func (impl ChartTemplateServiceImpl) FetchValuesFromReferenceChart(chartMetaData func (impl ChartTemplateServiceImpl) BuildChart(ctx context.Context, chartMetaData *chart.Metadata, referenceTemplatePath string) (string, error) { chartMetaData.ApiVersion = "v1" // ensure always v1 dir := impl.GetDir() - tempReferenceTemplateDir := filepath.Join(ChartWorkingDirPath, dir) + tempReferenceTemplateDir := filepath.Join(CHART_WORKING_DIR_PATH, dir) impl.logger.Debugw("chart dir ", "chart", chartMetaData.Name, "dir", tempReferenceTemplateDir) err := os.MkdirAll(tempReferenceTemplateDir, os.ModePerm) //hack for concurrency handling if err != nil { @@ -216,7 +216,7 @@ func (impl ChartTemplateServiceImpl) BuildChartProxyForHelmApps(chartCreateReque chartMetaData := chartCreateRequest.ChartMetaData chartMetaData.ApiVersion = "v2" // ensure always v2 dir := impl.GetDir() - chartDir := filepath.Join(ChartWorkingDirPath, dir) + chartDir := filepath.Join(CHART_WORKING_DIR_PATH, dir) impl.logger.Debugw("chart dir ", "chart", chartMetaData.Name, "dir", chartDir) err := os.MkdirAll(chartDir, os.ModePerm) //hack for concurrency handling if err != nil { @@ -482,7 +482,7 @@ func (impl ChartTemplateServiceImpl) GetDir() string { func (impl ChartTemplateServiceImpl) CreateChartProxy(chartMetaData *chart.Metadata, refChartLocation string, envName string, installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (string, *ChartGitAttribute, error) { chartMetaData.ApiVersion = "v2" // ensure always v2 dir := impl.GetDir() - chartDir := filepath.Join(ChartWorkingDirPath, dir) + chartDir := filepath.Join(CHART_WORKING_DIR_PATH, dir) impl.logger.Debugw("chart dir ", "chart", chartMetaData.Name, "dir", chartDir) err := os.MkdirAll(chartDir, os.ModePerm) //hack for concurrency handling if err != nil { @@ -664,7 +664,7 @@ func (impl ChartTemplateServiceImpl) GetGitOpsRepoNameFromUrl(gitRepoUrl string) func (impl ChartTemplateServiceImpl) GetByteArrayRefChart(chartMetaData *chart.Metadata, referenceTemplatePath string) ([]byte, error) { chartMetaData.ApiVersion = "v1" // ensure always v1 dir := impl.GetDir() - tempReferenceTemplateDir := filepath.Join(ChartWorkingDirPath, dir) + tempReferenceTemplateDir := filepath.Join(CHART_WORKING_DIR_PATH, dir) impl.logger.Debugw("chart dir ", "chart", chartMetaData.Name, "dir", tempReferenceTemplateDir) err := os.MkdirAll(tempReferenceTemplateDir, os.ModePerm) //hack for concurrency handling if err != nil { diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index 5a17c7dfcd..1e4b6cbb8b 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -22,6 +22,7 @@ import ( "encoding/json" "fmt" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" + bean3 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/bean" "io/ioutil" "net/url" "os" @@ -873,7 +874,7 @@ func (impl *AppServiceImpl) BuildChartAndGetPath(appName string, envOverride *ch Name: appName, Version: envOverride.Chart.ChartVersion, } - referenceTemplatePath := path.Join(chartRepoRepository.RefChartDirPath, envOverride.Chart.ReferenceTemplate) + referenceTemplatePath := path.Join(bean3.RefChartDirPath, envOverride.Chart.ReferenceTemplate) // Load custom charts to referenceTemplatePath if not exists if _, err := os.Stat(referenceTemplatePath); os.IsNotExist(err) { chartRefValue, err := impl.chartRefService.FindById(envOverride.Chart.ChartRefId) @@ -882,7 +883,7 @@ func (impl *AppServiceImpl) BuildChartAndGetPath(appName string, envOverride *ch return "", err } if chartRefValue.ChartData != nil { - chartInfo, err := impl.chartRefService.ExtractChartIfMissing(chartRefValue.ChartData, chartRepoRepository.RefChartDirPath, chartRefValue.Location) + chartInfo, err := impl.chartRefService.ExtractChartIfMissing(chartRefValue.ChartData, bean3.RefChartDirPath, chartRefValue.Location) if chartInfo != nil && chartInfo.TemporaryFolder != "" { err1 := os.RemoveAll(chartInfo.TemporaryFolder) if err1 != nil { diff --git a/pkg/chart/ChartService.go b/pkg/chart/ChartService.go index 536b336c37..58222ac4e2 100644 --- a/pkg/chart/ChartService.go +++ b/pkg/chart/ChartService.go @@ -168,7 +168,7 @@ func (impl ChartServiceImpl) Create(templateRequest TemplateRequest, ctx context return nil, err } - refChart, templateName, err, _, pipelineStrategyPath := impl.chartRefService.GetRefChart(templateRequest.ChartRefId) + refChart, templateName, _, pipelineStrategyPath, err := impl.chartRefService.GetRefChart(templateRequest.ChartRefId) if err != nil { return nil, err } @@ -334,7 +334,7 @@ func (impl ChartServiceImpl) CreateChartFromEnvOverride(templateRequest Template return nil, err } - refChart, templateName, err, _, pipelineStrategyPath := impl.chartRefService.GetRefChart(templateRequest.ChartRefId) + refChart, templateName, _, pipelineStrategyPath, err := impl.chartRefService.GetRefChart(templateRequest.ChartRefId) if err != nil { return nil, err } diff --git a/pkg/chartRepo/repository/ChartRefRepository.go b/pkg/chartRepo/repository/ChartRefRepository.go index 33f9d69fc5..b62a9d280f 100644 --- a/pkg/chartRepo/repository/ChartRefRepository.go +++ b/pkg/chartRepo/repository/ChartRefRepository.go @@ -7,10 +7,6 @@ import ( "strings" ) -const ( - RefChartDirPath = "scripts/devtron-reference-helm-charts" -) - type ChartRef struct { tableName struct{} `sql:"chart_ref" pg:",discard_unknown_columns"` Id int `sql:"id,pk"` @@ -44,7 +40,7 @@ type ChartRefRepository interface { CheckIfDataExists(location string) (bool, error) FetchChart(name string) ([]*ChartRef, error) FetchInfoOfChartConfiguredInApp(appId int) (*ChartRef, error) - FetchAllChartInfoByUploadFlag(userUploaded bool) ([]*ChartRef, error) + FetchAllNonUserUploadedChartInfo() ([]*ChartRef, error) } type ChartRefRepositoryImpl struct { dbConnection *pg.DB @@ -124,10 +120,10 @@ func (impl ChartRefRepositoryImpl) FetchChart(name string) ([]*ChartRef, error) return chartRefs, err } -func (impl ChartRefRepositoryImpl) FetchAllChartInfoByUploadFlag(userUploaded bool) ([]*ChartRef, error) { +func (impl ChartRefRepositoryImpl) FetchAllNonUserUploadedChartInfo() ([]*ChartRef, error) { var repo []*ChartRef err := impl.dbConnection.Model(&repo). - Where("user_uploaded = ?", userUploaded). + Where("user_uploaded = ?", false). Select() if err != nil { return repo, err diff --git a/pkg/deployment/manifest/deploymentTemplate/chartRef/ChartRefService.go b/pkg/deployment/manifest/deploymentTemplate/chartRef/ChartRefService.go index 2b10d904b6..58964b0876 100644 --- a/pkg/deployment/manifest/deploymentTemplate/chartRef/ChartRefService.go +++ b/pkg/deployment/manifest/deploymentTemplate/chartRef/ChartRefService.go @@ -24,6 +24,8 @@ import ( ) type ChartRefService interface { + //below methods are for getting data from db + GetDefault() (*bean.ChartRefDto, error) GetAll() ([]*bean.ChartRefDto, error) GetAllChartMetadata() (map[string]bean.ChartRefMetaData, error) @@ -31,22 +33,23 @@ type ChartRefService interface { FindByVersionAndName(version, name string) (*bean.ChartRefDto, error) FetchInfoOfChartConfiguredInApp(appId int) (*bean.ChartRefDto, error) ChartRefAutocomplete() ([]*bean.ChartRefAutocompleteDto, error) + CheckChartExists(chartRefId int) error + ChartRefIdsCompatible(oldChartRefId int, newChartRefId int) (bool, string, string) + + //below methods are for custom chart SaveCustomChart(req *bean.CustomChartRefDto) error FetchCustomChartsInfo() ([]*bean.ChartDto, error) ValidateCustomChartUploadedFileFormat(fileName string) error - GetAppOverrideForDefaultTemplate(chartRefId int) (map[string]interface{}, string, error) + //below methods are for chart file data actions + GetAppOverrideForDefaultTemplate(chartRefId int) (map[string]interface{}, string, error) JsonSchemaExtractFromFile(chartRefId int) (map[string]interface{}, string, error) GetSchemaAndReadmeForTemplateByChartRefId(chartRefId int) ([]byte, []byte, error) - - ChartRefIdsCompatible(oldChartRefId int, newChartRefId int) (bool, string, string) - - CheckChartExists(chartRefId int) error - GetRefChart(chartRefId int) (string, string, error, string, string) + GetRefChart(chartRefId int) (string, string, string, string, error) ExtractChartIfMissing(chartData []byte, refChartDir string, location string) (*bean.ChartDataInfo, error) - GetCustomChartInBytes(chartRefId int) ([]byte, error) + GetChartInBytes(chartRefId int) ([]byte, error) } type ChartRefServiceImpl struct { @@ -61,7 +64,7 @@ func NewChartRefServiceImpl(logger *zap.SugaredLogger, chartTemplateService util.ChartTemplateService, mergeUtil util.MergeUtil) *ChartRefServiceImpl { // cache devtron reference charts list - devtronChartList, _ := chartRefRepository.FetchAllChartInfoByUploadFlag(false) + devtronChartList, _ := chartRefRepository.FetchAllNonUserUploadedChartInfo() setReservedChartList(devtronChartList) return &ChartRefServiceImpl{ logger: logger, @@ -165,7 +168,7 @@ func (impl *ChartRefServiceImpl) SaveCustomChart(req *bean.CustomChartRefDto) er return nil } -func (impl *ChartRefServiceImpl) GetRefChart(chartRefId int) (string, string, error, string, string) { +func (impl *ChartRefServiceImpl) GetRefChart(chartRefId int) (string, string, string, string, error) { var template string var version string //path of file in chart from where strategy config is to be taken @@ -175,12 +178,12 @@ func (impl *ChartRefServiceImpl) GetRefChart(chartRefId int) (string, string, er if err != nil { chartRef, err = impl.chartRefRepository.GetDefault() if err != nil { - return "", "", err, "", "" + return "", "", "", "", err } } else if chartRef.UserUploaded { - refChartLocation := filepath.Join(chartRepoRepository.RefChartDirPath, chartRef.Location) + refChartLocation := filepath.Join(bean.RefChartDirPath, chartRef.Location) if _, err := os.Stat(refChartLocation); os.IsNotExist(err) { - chartInfo, err := impl.ExtractChartIfMissing(chartRef.ChartData, chartRepoRepository.RefChartDirPath, chartRef.Location) + chartInfo, err := impl.ExtractChartIfMissing(chartRef.ChartData, bean.RefChartDirPath, chartRef.Location) if chartInfo != nil && chartInfo.TemporaryFolder != "" { err1 := os.RemoveAll(chartInfo.TemporaryFolder) if err1 != nil { @@ -189,7 +192,7 @@ func (impl *ChartRefServiceImpl) GetRefChart(chartRefId int) (string, string, er } if err != nil { impl.logger.Errorw("Error regarding uploaded chart", "err", err) - return "", "", err, "", "" + return "", "", "", "", err } } @@ -200,7 +203,7 @@ func (impl *ChartRefServiceImpl) GetRefChart(chartRefId int) (string, string, er } else { chartRef, err := impl.chartRefRepository.GetDefault() if err != nil { - return "", "", err, "", "" + return "", "", "", "", err } template = chartRef.Location version = chartRef.Version @@ -208,17 +211,17 @@ func (impl *ChartRefServiceImpl) GetRefChart(chartRefId int) (string, string, er } //TODO VIKI- fetch from chart ref table - chartPath := path.Join(chartRepoRepository.RefChartDirPath, template) + chartPath := path.Join(bean.RefChartDirPath, template) valid, err := chartutil.IsChartDir(chartPath) if err != nil || !valid { impl.logger.Errorw("invalid base chart", "dir", chartPath, "err", err) - return "", "", err, "", "" + return "", "", "", "", err } - return chartPath, template, nil, version, pipelineStrategyPath + return chartPath, template, version, pipelineStrategyPath, nil } func (impl *ChartRefServiceImpl) GetSchemaAndReadmeForTemplateByChartRefId(chartRefId int) ([]byte, []byte, error) { - refChart, _, err, _, _ := impl.GetRefChart(chartRefId) + refChart, _, _, _, err := impl.GetRefChart(chartRefId) if err != nil { impl.logger.Errorw("error in getting refChart", "err", err, "chartRefId", chartRefId) return nil, nil, err @@ -262,7 +265,7 @@ func (impl *ChartRefServiceImpl) ChartRefAutocomplete() ([]*bean.ChartRefAutocom return chartRefs, nil } -func (impl *ChartRefServiceImpl) GetCustomChartInBytes(chartRefId int) ([]byte, error) { +func (impl *ChartRefServiceImpl) GetChartInBytes(chartRefId int) ([]byte, error) { chartRef, err := impl.chartRefRepository.FindById(chartRefId) if err != nil { impl.logger.Errorw("error getting chart data", "chartRefId", chartRefId, "err", err) @@ -273,7 +276,7 @@ func (impl *ChartRefServiceImpl) GetCustomChartInBytes(chartRefId int) ([]byte, return chartRef.ChartData, nil } // For Devtron reference charts the chart will be load from the directory location - refChartPath := filepath.Join(chartRepoRepository.RefChartDirPath, chartRef.Location) + refChartPath := filepath.Join(bean.RefChartDirPath, chartRef.Location) manifestByteArr, err := impl.chartTemplateService.LoadChartInBytes(refChartPath, false) if err != nil { impl.logger.Errorw("error in converting chart to bytes", "err", err) @@ -322,9 +325,9 @@ func (impl *ChartRefServiceImpl) CheckChartExists(chartRefId int) error { impl.logger.Errorw("error in finding ref chart by id", "err", err) return err } - refChartLocation := filepath.Join(chartRepoRepository.RefChartDirPath, chartRefValue.Location) + refChartLocation := filepath.Join(bean.RefChartDirPath, chartRefValue.Location) if _, err := os.Stat(refChartLocation); os.IsNotExist(err) { - chartInfo, err := impl.ExtractChartIfMissing(chartRefValue.ChartData, chartRepoRepository.RefChartDirPath, chartRefValue.Location) + chartInfo, err := impl.ExtractChartIfMissing(chartRefValue.ChartData, bean.RefChartDirPath, chartRefValue.Location) if chartInfo != nil && chartInfo.TemporaryFolder != "" { err1 := os.RemoveAll(chartInfo.TemporaryFolder) if err1 != nil { @@ -342,7 +345,7 @@ func (impl *ChartRefServiceImpl) GetAppOverrideForDefaultTemplate(chartRefId int return nil, "", err } - refChart, _, err, _, _ := impl.GetRefChart(chartRefId) + refChart, _, _, _, err := impl.GetRefChart(chartRefId) if err != nil { return nil, "", err } @@ -396,7 +399,7 @@ func (impl *ChartRefServiceImpl) JsonSchemaExtractFromFile(chartRefId int) (map[ return nil, "", err } - refChartDir, _, err, version, _ := impl.GetRefChart(chartRefId) + refChartDir, _, version, _, err := impl.GetRefChart(chartRefId) if err != nil { impl.logger.Errorw("refChartDir Not Found err, JsonSchemaExtractFromFile", err) return nil, "", err @@ -430,14 +433,7 @@ func (impl *ChartRefServiceImpl) JsonSchemaExtractFromFile(chartRefId int) (map[ func (impl *ChartRefServiceImpl) ExtractChartIfMissing(chartData []byte, refChartDir string, location string) (*bean.ChartDataInfo, error) { binaryDataReader := bytes.NewReader(chartData) dir := impl.chartTemplateService.GetDir() - chartInfo := &bean.ChartDataInfo{ - ChartName: "", - ChartVersion: "", - ChartLocation: "", - TemporaryFolder: "", - Description: "", - Message: "", - } + chartInfo := &bean.ChartDataInfo{} temporaryChartWorkingDir := filepath.Clean(filepath.Join(refChartDir, dir)) err := os.MkdirAll(temporaryChartWorkingDir, os.ModePerm) if err != nil { @@ -504,7 +500,7 @@ func (impl *ChartRefServiceImpl) ExtractChartIfMissing(chartData []byte, refChar impl.logger.Errorw("request err, chart name is reserved by Devtron") err = &util.ApiError{ Code: constants.ChartNameAlreadyReserved, - InternalMessage: bean.CHART_NAME_RESERVED_INTERNAL_ERROR, + InternalMessage: bean.ChartNameReservedInternalError, UserMessage: fmt.Sprintf("The name '%s' is reserved for a chart provided by Devtron", chartName), } return chartInfo, err @@ -520,7 +516,7 @@ func (impl *ChartRefServiceImpl) ExtractChartIfMissing(chartData []byte, refChar impl.logger.Errorw("request err, chart name and version exists already in the database") err = &util.ApiError{ Code: constants.ChartCreatedAlreadyExists, - InternalMessage: bean.CHART_ALREADY_EXISTS_INTERNAL_ERROR, + InternalMessage: bean.ChartAlreadyExistsInternalError, UserMessage: fmt.Sprintf("%s of %s exists already in the database", chartVersion, chartName), } return chartInfo, err diff --git a/pkg/deployment/manifest/deploymentTemplate/chartRef/bean/bean.go b/pkg/deployment/manifest/deploymentTemplate/chartRef/bean/bean.go index b0f662e04e..0a2428435e 100644 --- a/pkg/deployment/manifest/deploymentTemplate/chartRef/bean/bean.go +++ b/pkg/deployment/manifest/deploymentTemplate/chartRef/bean/bean.go @@ -3,11 +3,12 @@ package bean import "github.com/devtron-labs/devtron/pkg/sql" const ( - DeploymentChartType = "Deployment" - RolloutChartType = "Rollout Deployment" - ReferenceChart = "reference-chart" - CHART_ALREADY_EXISTS_INTERNAL_ERROR = "Chart exists already, try uploading another chart" - CHART_NAME_RESERVED_INTERNAL_ERROR = "Change the name of the chart and try uploading again" + DeploymentChartType = "Deployment" + RolloutChartType = "Rollout Deployment" + ReferenceChart = "reference-chart" + RefChartDirPath = "scripts/devtron-reference-helm-charts" + ChartAlreadyExistsInternalError = "Chart exists already, try uploading another chart" + ChartNameReservedInternalError = "Change the name of the chart and try uploading again" ) type ChartDataInfo struct { diff --git a/pkg/generateManifest/DeployementTemplateService.go b/pkg/generateManifest/DeployementTemplateService.go index b35924856a..5d5241d564 100644 --- a/pkg/generateManifest/DeployementTemplateService.go +++ b/pkg/generateManifest/DeployementTemplateService.go @@ -304,7 +304,7 @@ func (impl DeploymentTemplateServiceImpl) extractScopeData(request DeploymentTem } func (impl DeploymentTemplateServiceImpl) GenerateManifest(ctx context.Context, chartRefId int, valuesYaml string) (*openapi2.TemplateChartResponse, error) { - refChart, template, err, version, _ := impl.chartRefService.GetRefChart(chartRefId) + refChart, template, version, _, err := impl.chartRefService.GetRefChart(chartRefId) if err != nil { impl.Logger.Errorw("error in getting refChart", "err", err, "chartRefId", chartRefId) return nil, err diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index fd54856752..f32c6b997f 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -24,6 +24,7 @@ import ( "fmt" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" + bean5 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/bean" "path" "strconv" "strings" @@ -3499,7 +3500,7 @@ func (impl *WorkflowDagExecutorImpl) createHelmAppForCdPipeline(overrideRequest Name: pipeline.App.AppName, Version: envOverride.Chart.ChartVersion, } - referenceTemplatePath := path.Join(chartRepoRepository.RefChartDirPath, envOverride.Chart.ReferenceTemplate) + referenceTemplatePath := path.Join(bean5.RefChartDirPath, envOverride.Chart.ReferenceTemplate) if util.IsHelmApp(pipeline.DeploymentAppType) { referenceChartByte := envOverride.Chart.ReferenceChart From ab494c1cf6c9000ef06290f1ae21456d4e73b235 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Fri, 19 Jan 2024 12:40:43 +0530 Subject: [PATCH 27/64] wip --- pkg/pipeline/PropertiesConfig.go | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/pkg/pipeline/PropertiesConfig.go b/pkg/pipeline/PropertiesConfig.go index 210bef4093..c9792a9240 100644 --- a/pkg/pipeline/PropertiesConfig.go +++ b/pkg/pipeline/PropertiesConfig.go @@ -367,6 +367,23 @@ func (impl PropertiesConfigServiceImpl) CreateIfRequired(chart *chartRepoReposit namespace = env.Namespace } + if isOverride { //case of override, to do app metrics operation + envLevelMetricsUpdateReq := &bean2.DeployedAppMetricsRequest{ + EnableMetrics: isAppMetricsEnabled, + AppId: chart.AppId, + EnvId: environmentId, + ChartRefId: chart.ChartRefId, + UserId: userId, + } + err = impl.deployedAppMetricsService.CreateOrUpdateAppOrEnvLevelMetrics(context.Background(), envLevelMetricsUpdateReq) + if err != nil { + impl.logger.Errorw("error, CheckAndUpdateAppOrEnvLevelMetrics", "err", err, "req", envLevelMetricsUpdateReq) + return nil, isAppMetricsEnabled, err + } + //updating metrics flag because it might be possible that the chartRef used was not supported and that could have override the metrics flag got in request + isAppMetricsEnabled = envLevelMetricsUpdateReq.EnableMetrics + } + envOverride, err := impl.envConfigRepo.GetByChartAndEnvironment(chart.Id, environmentId) if err != nil && !errors.IsNotFound(err) { return nil, isAppMetricsEnabled, err @@ -427,20 +444,6 @@ func (impl PropertiesConfigServiceImpl) CreateIfRequired(chart *chartRepoReposit impl.logger.Errorw("error in creating envconfig", "data", envOverride, "error", err) return nil, isAppMetricsEnabled, err } - envLevelMetricsUpdateReq := &bean2.DeployedAppMetricsRequest{ - EnableMetrics: isAppMetricsEnabled, - AppId: chart.AppId, - EnvId: environmentId, - ChartRefId: chart.ChartRefId, - UserId: userId, - } - err = impl.deployedAppMetricsService.CreateOrUpdateAppOrEnvLevelMetrics(context.Background(), envLevelMetricsUpdateReq) - if err != nil { - impl.logger.Errorw("error, CheckAndUpdateAppOrEnvLevelMetrics", "err", err, "req", envLevelMetricsUpdateReq) - return nil, isAppMetricsEnabled, err - } - //updating metrics flag because it might be possible that the chartRef used was not supported and that could have override the metrics flag got in request - isAppMetricsEnabled = envLevelMetricsUpdateReq.EnableMetrics err = impl.deploymentTemplateHistoryService.CreateDeploymentTemplateHistoryFromEnvOverrideTemplate(envOverride, tx, isAppMetricsEnabled, 0) if err != nil { impl.logger.Errorw("error in creating entry for env deployment template history", "err", err, "envOverride", envOverride) From 0120e15600328cc1e6380c4e4b757efba2fcbc81 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Fri, 19 Jan 2024 19:57:51 +0530 Subject: [PATCH 28/64] fix for unsupported charts --- pkg/deployment/manifest/deployedAppMetrics/DeployedAppMetrics.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/deployment/manifest/deployedAppMetrics/DeployedAppMetrics.go b/pkg/deployment/manifest/deployedAppMetrics/DeployedAppMetrics.go index 2144b71cd5..f866a09825 100644 --- a/pkg/deployment/manifest/deployedAppMetrics/DeployedAppMetrics.go +++ b/pkg/deployment/manifest/deployedAppMetrics/DeployedAppMetrics.go @@ -95,7 +95,6 @@ func (impl *DeployedAppMetricsServiceImpl) CreateOrUpdateAppOrEnvLevelMetrics(ct if !(isAppMetricsSupported) { //chart does not have metrics support, disabling req.EnableMetrics = false - return nil } if req.EnvId == 0 { _, span := otel.Tracer("orchestrator").Start(ctx, "createOrUpdateAppLevelMetrics") From 0b44511ad33fcbcc26411bfe7f32957c668c8746 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Sun, 21 Jan 2024 02:50:56 +0530 Subject: [PATCH 29/64] refactoring: App Store deployment services --- Wire.go | 3 - api/appStore/InstalledAppRestHandler.go | 2 +- .../AppStoreDeploymentRestHandler.go | 5 +- .../deployment/CommonDeploymentRestHandler.go | 50 ++- api/bean/GitOpsConfig.go | 1 + api/helm-app/HelmAppRestHandler.go | 120 ++----- api/helm-app/bean.go | 24 -- api/helm-app/bean/bean.go | 82 +++++ api/helm-app/{ => gRPC}/applicationClient.go | 11 +- api/helm-app/{ => gRPC}/applist.pb.go | 2 +- api/helm-app/{ => gRPC}/applist.proto | 0 api/helm-app/{ => gRPC}/applist_grpc.pb.go | 2 +- api/helm-app/mocks/HelmAppClient.go | 220 ++++++------ api/helm-app/mocks/HelmAppService.go | 109 +++--- api/helm-app/{ => service}/HelmAppService.go | 146 ++++---- api/helm-app/wire_helmApp.go | 14 +- .../application/k8sApplicationRestHandler.go | 2 +- api/restHandler/AppListingRestHandler.go | 28 +- api/restHandler/AppRestHandler.go | 2 +- client/telemetry/TelemetryEventClient.go | 10 +- .../telemetry/TelemetryEventClientExtended.go | 2 +- cmd/external-app/wire_gen.go | 62 ++-- internal/util/ChartService.go | 28 +- pkg/app/integrationTest/AppService_test.go | 7 +- pkg/appStore/bean/bean.go | 9 - pkg/appStore/deployment/adapter/Adapter.go | 26 ++ .../common/AppStoreDeploymentCommonService.go | 259 +------------- .../common/AppStoreDeploymentGitService.go | 184 ---------- .../AppStoreDeploymentFullModeService.go | 289 --------------- .../service/AppStoreDeploymentDBService.go | 32 ++ .../service/AppStoreDeploymentService.go | 93 ++--- .../service/AppStoreDeploymentService_test.go | 2 +- .../deployment/service/InstalledAppService.go | 84 +++-- pkg/appStore/deployment/service/Notes.go | 8 +- .../deployment/service/ResourceTree.go | 4 +- .../tool/AppStoreDeploymentArgoCdService.go | 336 +++++++++++------- .../tool/AppStoreDeploymentGitOpsService.go | 272 ++++++++++++++ .../tool/AppStoreDeploymentHelmService.go | 135 +++---- pkg/appStore/deployment/tool/bean/bean.go | 17 + pkg/bulkAction/BulkUpdateService.go | 2 +- .../UserTerminalAccessService.go | 2 +- .../remote/GitOpsRemoteOperationService.go | 148 +++++--- pkg/deployment/gitOps/remote/bean/bean.go | 9 + .../DeployementTemplateService.go | 18 +- .../DeployementTemplateService_test.go | 2 +- pkg/k8s/K8sCommonService.go | 4 +- pkg/k8s/application/k8sApplicationService.go | 5 +- .../application/k8sApplicationService_test.go | 2 +- .../mocks/K8sApplicationService.go | 2 +- pkg/k8s/bean.go | 2 +- .../kubernetesResourceHistoryService.go | 2 +- pkg/module/ModuleCronService.go | 23 +- pkg/module/ModuleService.go | 5 +- .../AppDeploymentTypeChangeManager.go | 10 +- pkg/pipeline/CdHandler.go | 2 +- .../DeploymentPipelineConfigService.go | 2 +- pkg/pipeline/DockerRegistryConfig.go | 5 +- pkg/pipeline/DockerRegistryConfig_test.go | 2 +- pkg/pipeline/WorkflowDagExecutor.go | 47 +-- pkg/server/ServerCacheService.go | 2 +- pkg/server/ServerService.go | 5 +- pkg/webhook/helm/WebhookHelmService.go | 14 +- wire_gen.go | 38 +- 63 files changed, 1439 insertions(+), 1596 deletions(-) delete mode 100644 api/helm-app/bean.go create mode 100644 api/helm-app/bean/bean.go rename api/helm-app/{ => gRPC}/applicationClient.go (96%) rename api/helm-app/{ => gRPC}/applist.pb.go (99%) rename api/helm-app/{ => gRPC}/applist.proto (100%) rename api/helm-app/{ => gRPC}/applist_grpc.pb.go (99%) rename api/helm-app/{ => service}/HelmAppService.go (89%) create mode 100644 pkg/appStore/deployment/adapter/Adapter.go delete mode 100644 pkg/appStore/deployment/common/AppStoreDeploymentGitService.go delete mode 100644 pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go create mode 100644 pkg/appStore/deployment/tool/AppStoreDeploymentGitOpsService.go create mode 100644 pkg/appStore/deployment/tool/bean/bean.go diff --git a/Wire.go b/Wire.go index 7c6e26c779..50715f0cb8 100644 --- a/Wire.go +++ b/Wire.go @@ -87,7 +87,6 @@ import ( "github.com/devtron-labs/devtron/pkg/appStatus" "github.com/devtron-labs/devtron/pkg/appStore/chartGroup" repository4 "github.com/devtron-labs/devtron/pkg/appStore/chartGroup/repository" - appStoreDeploymentFullMode "github.com/devtron-labs/devtron/pkg/appStore/deployment/fullMode" "github.com/devtron-labs/devtron/pkg/appStore/deployment/service" appStoreDeploymentGitopsTool "github.com/devtron-labs/devtron/pkg/appStore/deployment/tool" "github.com/devtron-labs/devtron/pkg/appWorkflow" @@ -713,8 +712,6 @@ func InitializeApp() (*App, error) { delete2.NewDeleteServiceFullModeImpl, wire.Bind(new(delete2.DeleteServiceFullMode), new(*delete2.DeleteServiceFullModeImpl)), - appStoreDeploymentFullMode.NewAppStoreDeploymentFullModeServiceImpl, - wire.Bind(new(appStoreDeploymentFullMode.AppStoreDeploymentFullModeService), new(*appStoreDeploymentFullMode.AppStoreDeploymentFullModeServiceImpl)), appStoreDeploymentGitopsTool.NewAppStoreDeploymentArgoCdServiceImpl, wire.Bind(new(appStoreDeploymentGitopsTool.AppStoreDeploymentArgoCdService), new(*appStoreDeploymentGitopsTool.AppStoreDeploymentArgoCdServiceImpl)), // util2.NewGoJsonSchemaCustomFormatChecker, diff --git a/api/appStore/InstalledAppRestHandler.go b/api/appStore/InstalledAppRestHandler.go index c6e1427c8d..94399e6a38 100644 --- a/api/appStore/InstalledAppRestHandler.go +++ b/api/appStore/InstalledAppRestHandler.go @@ -22,13 +22,13 @@ import ( "encoding/json" "errors" "fmt" + client "github.com/devtron-labs/devtron/api/helm-app/gRPC" "net/http" "strconv" "strings" "time" bean2 "github.com/devtron-labs/devtron/api/bean" - client "github.com/devtron-labs/devtron/api/helm-app" openapi "github.com/devtron-labs/devtron/api/helm-app/openapiClient" "github.com/devtron-labs/devtron/api/restHandler/common" "github.com/devtron-labs/devtron/client/argocdServer/application" diff --git a/api/appStore/deployment/AppStoreDeploymentRestHandler.go b/api/appStore/deployment/AppStoreDeploymentRestHandler.go index cffb21ff55..ef8e9a28e6 100644 --- a/api/appStore/deployment/AppStoreDeploymentRestHandler.go +++ b/api/appStore/deployment/AppStoreDeploymentRestHandler.go @@ -22,6 +22,7 @@ import ( "encoding/json" "errors" "fmt" + service2 "github.com/devtron-labs/devtron/api/helm-app/service" "net/http" "strconv" "strings" @@ -67,7 +68,7 @@ type AppStoreDeploymentRestHandlerImpl struct { appStoreDeploymentService service.AppStoreDeploymentService appStoreDeploymentServiceC appStoreDeploymentCommon.AppStoreDeploymentCommonService validator *validator.Validate - helmAppService client.HelmAppService + helmAppService service2.HelmAppService helmAppRestHandler client.HelmAppRestHandler argoUserService argo.ArgoUserService attributesService attributes.AttributesService @@ -75,7 +76,7 @@ type AppStoreDeploymentRestHandlerImpl struct { func NewAppStoreDeploymentRestHandlerImpl(Logger *zap.SugaredLogger, userAuthService user.UserService, enforcer casbin.Enforcer, enforcerUtil rbac.EnforcerUtil, enforcerUtilHelm rbac.EnforcerUtilHelm, appStoreDeploymentService service.AppStoreDeploymentService, - validator *validator.Validate, helmAppService client.HelmAppService, appStoreDeploymentServiceC appStoreDeploymentCommon.AppStoreDeploymentCommonService, + validator *validator.Validate, helmAppService service2.HelmAppService, appStoreDeploymentServiceC appStoreDeploymentCommon.AppStoreDeploymentCommonService, argoUserService argo.ArgoUserService, attributesService attributes.AttributesService) *AppStoreDeploymentRestHandlerImpl { return &AppStoreDeploymentRestHandlerImpl{ Logger: Logger, diff --git a/api/appStore/deployment/CommonDeploymentRestHandler.go b/api/appStore/deployment/CommonDeploymentRestHandler.go index d7bfac9fd8..0a4e527834 100644 --- a/api/appStore/deployment/CommonDeploymentRestHandler.go +++ b/api/appStore/deployment/CommonDeploymentRestHandler.go @@ -21,6 +21,7 @@ import ( "context" "encoding/json" "fmt" + service2 "github.com/devtron-labs/devtron/api/helm-app/service" "net/http" "strconv" "time" @@ -31,7 +32,6 @@ import ( "github.com/devtron-labs/devtron/api/restHandler/common" "github.com/devtron-labs/devtron/internal/util" appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" - appStoreDeploymentCommon "github.com/devtron-labs/devtron/pkg/appStore/deployment/common" "github.com/devtron-labs/devtron/pkg/appStore/deployment/service" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" "github.com/devtron-labs/devtron/pkg/auth/user" @@ -51,35 +51,33 @@ type CommonDeploymentRestHandler interface { } type CommonDeploymentRestHandlerImpl struct { - Logger *zap.SugaredLogger - userAuthService user.UserService - enforcer casbin.Enforcer - enforcerUtil rbac.EnforcerUtil - enforcerUtilHelm rbac.EnforcerUtilHelm - appStoreDeploymentService service.AppStoreDeploymentService - appStoreDeploymentServiceC appStoreDeploymentCommon.AppStoreDeploymentCommonService - validator *validator.Validate - helmAppService client.HelmAppService - helmAppRestHandler client.HelmAppRestHandler - argoUserService argo.ArgoUserService + Logger *zap.SugaredLogger + userAuthService user.UserService + enforcer casbin.Enforcer + enforcerUtil rbac.EnforcerUtil + enforcerUtilHelm rbac.EnforcerUtilHelm + appStoreDeploymentService service.AppStoreDeploymentService + validator *validator.Validate + helmAppService service2.HelmAppService + helmAppRestHandler client.HelmAppRestHandler + argoUserService argo.ArgoUserService } func NewCommonDeploymentRestHandlerImpl(Logger *zap.SugaredLogger, userAuthService user.UserService, enforcer casbin.Enforcer, enforcerUtil rbac.EnforcerUtil, enforcerUtilHelm rbac.EnforcerUtilHelm, appStoreDeploymentService service.AppStoreDeploymentService, - validator *validator.Validate, helmAppService client.HelmAppService, appStoreDeploymentServiceC appStoreDeploymentCommon.AppStoreDeploymentCommonService, + validator *validator.Validate, helmAppService service2.HelmAppService, helmAppRestHandler client.HelmAppRestHandler, argoUserService argo.ArgoUserService) *CommonDeploymentRestHandlerImpl { return &CommonDeploymentRestHandlerImpl{ - Logger: Logger, - userAuthService: userAuthService, - enforcer: enforcer, - enforcerUtil: enforcerUtil, - enforcerUtilHelm: enforcerUtilHelm, - appStoreDeploymentService: appStoreDeploymentService, - validator: validator, - helmAppService: helmAppService, - appStoreDeploymentServiceC: appStoreDeploymentServiceC, - helmAppRestHandler: helmAppRestHandler, - argoUserService: argoUserService, + Logger: Logger, + userAuthService: userAuthService, + enforcer: enforcer, + enforcerUtil: enforcerUtil, + enforcerUtilHelm: enforcerUtilHelm, + appStoreDeploymentService: appStoreDeploymentService, + validator: validator, + helmAppService: helmAppService, + helmAppRestHandler: helmAppRestHandler, + argoUserService: argoUserService, } } func (handler *CommonDeploymentRestHandlerImpl) getAppOfferingMode(installedAppId string, appId string) (string, *appStoreBean.InstallAppVersionDTO, error) { @@ -91,7 +89,7 @@ func (handler *CommonDeploymentRestHandlerImpl) getAppOfferingMode(installedAppI err = &util.ApiError{HttpStatusCode: http.StatusBadRequest, UserMessage: "invalid app id"} return appOfferingMode, installedAppDto, err } - installedAppDto, err = handler.appStoreDeploymentServiceC.GetInstalledAppByClusterNamespaceAndName(appIdentifier.ClusterId, appIdentifier.Namespace, appIdentifier.ReleaseName) + installedAppDto, err = handler.appStoreDeploymentService.GetInstalledAppByClusterNamespaceAndName(appIdentifier.ClusterId, appIdentifier.Namespace, appIdentifier.ReleaseName) if err != nil { err = &util.ApiError{HttpStatusCode: http.StatusInternalServerError, UserMessage: "unable to find app in database"} return appOfferingMode, installedAppDto, err @@ -117,7 +115,7 @@ func (handler *CommonDeploymentRestHandlerImpl) getAppOfferingMode(installedAppI err = &util.ApiError{HttpStatusCode: http.StatusBadRequest, UserMessage: "invalid installed app id"} return appOfferingMode, installedAppDto, err } - installedAppDto, err = handler.appStoreDeploymentServiceC.GetInstalledAppByInstalledAppId(installedAppId) + installedAppDto, err = handler.appStoreDeploymentService.GetInstalledAppByInstalledAppId(installedAppId) if err != nil { err = &util.ApiError{HttpStatusCode: http.StatusInternalServerError, UserMessage: "unable to find app in database"} return appOfferingMode, installedAppDto, err diff --git a/api/bean/GitOpsConfig.go b/api/bean/GitOpsConfig.go index 4ff2d5bd92..1876c63fda 100644 --- a/api/bean/GitOpsConfig.go +++ b/api/bean/GitOpsConfig.go @@ -13,6 +13,7 @@ type GitOpsConfigDto struct { BitBucketWorkspaceId string `json:"bitBucketWorkspaceId"` BitBucketProjectKey string `json:"bitBucketProjectKey"` + // TODO refactoring: create different struct for internal fields GitRepoName string `json:"gitRepoName"` UserEmailId string `json:"userEmailId"` Description string `json:"description"` diff --git a/api/helm-app/HelmAppRestHandler.go b/api/helm-app/HelmAppRestHandler.go index 9259347308..45e504c08f 100644 --- a/api/helm-app/HelmAppRestHandler.go +++ b/api/helm-app/HelmAppRestHandler.go @@ -4,6 +4,9 @@ import ( "context" "encoding/json" "errors" + "github.com/devtron-labs/devtron/api/helm-app/bean" + service2 "github.com/devtron-labs/devtron/api/helm-app/service" + "github.com/devtron-labs/devtron/pkg/appStore/deployment/service" "net/http" "strconv" "strings" @@ -14,8 +17,6 @@ import ( openapi2 "github.com/devtron-labs/devtron/api/openapi/openapiClient" "github.com/devtron-labs/devtron/api/restHandler/common" "github.com/devtron-labs/devtron/internal/util" - appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" - appStoreDeploymentCommon "github.com/devtron-labs/devtron/pkg/appStore/deployment/common" "github.com/devtron-labs/devtron/pkg/attributes" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" "github.com/devtron-labs/devtron/pkg/auth/user" @@ -43,31 +44,32 @@ const HELM_APP_ACCESS_COUNTER = "HelmAppAccessCounter" const HELM_APP_UPDATE_COUNTER = "HelmAppUpdateCounter" type HelmAppRestHandlerImpl struct { - logger *zap.SugaredLogger - helmAppService HelmAppService - enforcer casbin.Enforcer - clusterService cluster.ClusterService - enforcerUtil rbac.EnforcerUtilHelm - appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService - userAuthService user.UserService - attributesService attributes.AttributesService - serverEnvConfig *serverEnvConfig.ServerEnvConfig + logger *zap.SugaredLogger + helmAppService service2.HelmAppService + enforcer casbin.Enforcer + clusterService cluster.ClusterService + enforcerUtil rbac.EnforcerUtilHelm + appStoreDeploymentService service.AppStoreDeploymentService + userAuthService user.UserService + attributesService attributes.AttributesService + serverEnvConfig *serverEnvConfig.ServerEnvConfig } func NewHelmAppRestHandlerImpl(logger *zap.SugaredLogger, - helmAppService HelmAppService, enforcer casbin.Enforcer, - clusterService cluster.ClusterService, enforcerUtil rbac.EnforcerUtilHelm, appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService, + helmAppService service2.HelmAppService, enforcer casbin.Enforcer, + clusterService cluster.ClusterService, enforcerUtil rbac.EnforcerUtilHelm, + appStoreDeploymentService service.AppStoreDeploymentService, userAuthService user.UserService, attributesService attributes.AttributesService, serverEnvConfig *serverEnvConfig.ServerEnvConfig) *HelmAppRestHandlerImpl { return &HelmAppRestHandlerImpl{ - logger: logger, - helmAppService: helmAppService, - enforcer: enforcer, - clusterService: clusterService, - enforcerUtil: enforcerUtil, - appStoreDeploymentCommonService: appStoreDeploymentCommonService, - userAuthService: userAuthService, - attributesService: attributesService, - serverEnvConfig: serverEnvConfig, + logger: logger, + helmAppService: helmAppService, + enforcer: enforcer, + clusterService: clusterService, + enforcerUtil: enforcerUtil, + appStoreDeploymentService: appStoreDeploymentService, + userAuthService: userAuthService, + attributesService: attributesService, + serverEnvConfig: serverEnvConfig, } } @@ -118,15 +120,15 @@ func (handler *HelmAppRestHandlerImpl) GetApplicationDetail(w http.ResponseWrite return } - installedApp, err := handler.appStoreDeploymentCommonService.GetInstalledAppByClusterNamespaceAndName(appIdentifier.ClusterId, appIdentifier.Namespace, appIdentifier.ReleaseName) + installedApp, err := handler.appStoreDeploymentService.GetInstalledAppByClusterNamespaceAndName(appIdentifier.ClusterId, appIdentifier.Namespace, appIdentifier.ReleaseName) if err != nil { common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } - res := &AppDetailAndInstalledAppInfo{ + res := &bean.AppDetailAndInstalledAppInfo{ AppDetail: appdetail, - InstalledAppInfo: convertToInstalledAppInfo(installedApp), + InstalledAppInfo: bean.ConvertToInstalledAppInfo(installedApp), } common.WriteJsonResp(w, err, res, http.StatusOK) @@ -219,15 +221,15 @@ func (handler *HelmAppRestHandlerImpl) GetReleaseInfo(w http.ResponseWriter, r * common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } - - installedApp, err := handler.appStoreDeploymentCommonService.GetInstalledAppByClusterNamespaceAndName(appIdentifier.ClusterId, appIdentifier.Namespace, appIdentifier.ReleaseName) + installedApp, err := handler.appStoreDeploymentService.GetInstalledAppByClusterNamespaceAndName(appIdentifier.ClusterId, appIdentifier.Namespace, appIdentifier.ReleaseName) if err != nil { common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } - res := &ReleaseAndInstalledAppInfo{ + + res := &bean.ReleaseAndInstalledAppInfo{ ReleaseInfo: releaseInfo, - InstalledAppInfo: convertToInstalledAppInfo(installedApp), + InstalledAppInfo: bean.ConvertToInstalledAppInfo(installedApp), } common.WriteJsonResp(w, err, res, http.StatusOK) @@ -306,7 +308,7 @@ func (handler *HelmAppRestHandlerImpl) DeleteApplication(w http.ResponseWriter, // validate if the devtron-operator helm release, block that for deletion if appIdentifier.ReleaseName == handler.serverEnvConfig.DevtronHelmReleaseName && appIdentifier.Namespace == handler.serverEnvConfig.DevtronHelmReleaseNamespace && - appIdentifier.ClusterId == DEFAULT_CLUSTER_ID { + appIdentifier.ClusterId == bean.DEFAULT_CLUSTER_ID { common.WriteJsonResp(w, errors.New("cannot delete this default helm app"), nil, http.StatusForbidden) return } @@ -323,7 +325,7 @@ func (handler *HelmAppRestHandlerImpl) DeleteApplication(w http.ResponseWriter, } func (handler *HelmAppRestHandlerImpl) UpdateApplication(w http.ResponseWriter, r *http.Request) { - request := &UpdateApplicationRequestDto{} + request := &bean.UpdateApplicationRequestDto{} decoder := json.NewDecoder(r.Body) err := decoder.Decode(request) if err != nil { @@ -346,7 +348,7 @@ func (handler *HelmAppRestHandlerImpl) UpdateApplication(w http.ResponseWriter, return } //RBAC enforcer Ends - request.SourceAppType = SOURCE_EXTERNAL_HELM_APP + request.SourceAppType = bean.SOURCE_EXTERNAL_HELM_APP // update application externally res, err := handler.helmAppService.UpdateApplication(r.Context(), appIdentifier, request) if err != nil { @@ -422,57 +424,3 @@ func (handler *HelmAppRestHandlerImpl) SaveHelmAppDetailsViewedTelemetryData(w h common.WriteJsonResp(w, err, nil, http.StatusOK) } - -func convertToInstalledAppInfo(installedApp *appStoreBean.InstallAppVersionDTO) *InstalledAppInfo { - if installedApp == nil { - return nil - } - - chartInfo := installedApp.InstallAppVersionChartDTO - - return &InstalledAppInfo{ - AppId: installedApp.AppId, - EnvironmentName: installedApp.EnvironmentName, - AppOfferingMode: installedApp.AppOfferingMode, - InstalledAppId: installedApp.InstalledAppId, - InstalledAppVersionId: installedApp.InstalledAppVersionId, - AppStoreChartId: chartInfo.AppStoreChartId, - ClusterId: installedApp.ClusterId, - EnvironmentId: installedApp.EnvironmentId, - AppStoreChartRepoName: chartInfo.InstallAppVersionChartRepoDTO.RepoName, - AppStoreChartName: chartInfo.ChartName, - TeamId: installedApp.TeamId, - TeamName: installedApp.TeamName, - } -} - -type AppDetailAndInstalledAppInfo struct { - InstalledAppInfo *InstalledAppInfo `json:"installedAppInfo"` - AppDetail *AppDetail `json:"appDetail"` -} - -type ReleaseAndInstalledAppInfo struct { - InstalledAppInfo *InstalledAppInfo `json:"installedAppInfo"` - ReleaseInfo *ReleaseInfo `json:"releaseInfo"` -} - -type DeploymentHistoryAndInstalledAppInfo struct { - InstalledAppInfo *InstalledAppInfo `json:"installedAppInfo"` - DeploymentHistory []*HelmAppDeploymentDetail `json:"deploymentHistory"` -} - -type InstalledAppInfo struct { - AppId int `json:"appId"` - InstalledAppId int `json:"installedAppId"` - InstalledAppVersionId int `json:"installedAppVersionId"` - AppStoreChartId int `json:"appStoreChartId"` - EnvironmentName string `json:"environmentName"` - AppOfferingMode string `json:"appOfferingMode"` - ClusterId int `json:"clusterId"` - EnvironmentId int `json:"environmentId"` - AppStoreChartRepoName string `json:"appStoreChartRepoName"` - AppStoreChartName string `json:"appStoreChartName"` - TeamId int `json:"teamId"` - TeamName string `json:"teamName"` - DeploymentType string `json:"deploymentType"` -} diff --git a/api/helm-app/bean.go b/api/helm-app/bean.go deleted file mode 100644 index bd006b6e36..0000000000 --- a/api/helm-app/bean.go +++ /dev/null @@ -1,24 +0,0 @@ -package client - -import openapi "github.com/devtron-labs/devtron/api/helm-app/openapiClient" - -const ( - DEFAULT_CLUSTER_ID = 1 - SOURCE_DEVTRON_APP SourceAppType = "devtron-app" - SOURCE_HELM_APP SourceAppType = "helm-app" - SOURCE_EXTERNAL_HELM_APP SourceAppType = "external-helm-app" - SOURCE_UNKNOWN SourceAppType = "unknown" - ErrReleaseNotFound string = "release: not found" -) - -type SourceAppType string - -type UpdateApplicationRequestDto struct { - *openapi.UpdateReleaseRequest - SourceAppType SourceAppType `json:"-"` -} - -type UpdateApplicationWithChartInfoRequestDto struct { - *InstallReleaseRequest - SourceAppType SourceAppType `json:"-"` -} diff --git a/api/helm-app/bean/bean.go b/api/helm-app/bean/bean.go new file mode 100644 index 0000000000..f1222946e5 --- /dev/null +++ b/api/helm-app/bean/bean.go @@ -0,0 +1,82 @@ +package bean + +import ( + "github.com/devtron-labs/devtron/api/helm-app/gRPC" + openapi "github.com/devtron-labs/devtron/api/helm-app/openapiClient" + "github.com/devtron-labs/devtron/pkg/appStore/bean" +) + +const ( + DEFAULT_CLUSTER_ID = 1 + SOURCE_DEVTRON_APP SourceAppType = "devtron-app" + SOURCE_HELM_APP SourceAppType = "helm-app" + SOURCE_EXTERNAL_HELM_APP SourceAppType = "external-helm-app" + SOURCE_UNKNOWN SourceAppType = "unknown" + ErrReleaseNotFound string = "release: not found" +) + +type SourceAppType string + +type UpdateApplicationRequestDto struct { + *openapi.UpdateReleaseRequest + SourceAppType SourceAppType `json:"-"` +} + +type UpdateApplicationWithChartInfoRequestDto struct { + *gRPC.InstallReleaseRequest + SourceAppType SourceAppType `json:"-"` +} + +func ConvertToInstalledAppInfo(installedApp *appStoreBean.InstallAppVersionDTO) *InstalledAppInfo { + if installedApp == nil { + return nil + } + + chartInfo := installedApp.InstallAppVersionChartDTO + + return &InstalledAppInfo{ + AppId: installedApp.AppId, + EnvironmentName: installedApp.EnvironmentName, + AppOfferingMode: installedApp.AppOfferingMode, + InstalledAppId: installedApp.InstalledAppId, + InstalledAppVersionId: installedApp.InstalledAppVersionId, + AppStoreChartId: chartInfo.AppStoreChartId, + ClusterId: installedApp.ClusterId, + EnvironmentId: installedApp.EnvironmentId, + AppStoreChartRepoName: chartInfo.InstallAppVersionChartRepoDTO.RepoName, + AppStoreChartName: chartInfo.ChartName, + TeamId: installedApp.TeamId, + TeamName: installedApp.TeamName, + } +} + +type AppDetailAndInstalledAppInfo struct { + InstalledAppInfo *InstalledAppInfo `json:"installedAppInfo"` + AppDetail *gRPC.AppDetail `json:"appDetail"` +} + +type ReleaseAndInstalledAppInfo struct { + InstalledAppInfo *InstalledAppInfo `json:"installedAppInfo"` + ReleaseInfo *gRPC.ReleaseInfo `json:"releaseInfo"` +} + +type DeploymentHistoryAndInstalledAppInfo struct { + InstalledAppInfo *InstalledAppInfo `json:"installedAppInfo"` + DeploymentHistory []*gRPC.HelmAppDeploymentDetail `json:"deploymentHistory"` +} + +type InstalledAppInfo struct { + AppId int `json:"appId"` + InstalledAppId int `json:"installedAppId"` + InstalledAppVersionId int `json:"installedAppVersionId"` + AppStoreChartId int `json:"appStoreChartId"` + EnvironmentName string `json:"environmentName"` + AppOfferingMode string `json:"appOfferingMode"` + ClusterId int `json:"clusterId"` + EnvironmentId int `json:"environmentId"` + AppStoreChartRepoName string `json:"appStoreChartRepoName"` + AppStoreChartName string `json:"appStoreChartName"` + TeamId int `json:"teamId"` + TeamName string `json:"teamName"` + DeploymentType string `json:"deploymentType"` +} diff --git a/api/helm-app/applicationClient.go b/api/helm-app/gRPC/applicationClient.go similarity index 96% rename from api/helm-app/applicationClient.go rename to api/helm-app/gRPC/applicationClient.go index 884f088f1b..7f540df584 100644 --- a/api/helm-app/applicationClient.go +++ b/api/helm-app/gRPC/applicationClient.go @@ -1,11 +1,9 @@ -package client +package gRPC import ( "context" "fmt" "github.com/caarlos0/env" - "github.com/devtron-labs/devtron/internal/constants" - "github.com/devtron-labs/devtron/internal/util" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "go.uber.org/zap" "google.golang.org/grpc" @@ -158,13 +156,6 @@ func (impl *HelmAppClientImpl) GetDeploymentHistory(ctx context.Context, in *App } history, err := applicationClient.GetDeploymentHistory(ctx, in) if err != nil { - if util.GetGRPCErrorDetailedMessage(err) == ErrReleaseNotFound { - err = &util.ApiError{ - Code: constants.HelmReleaseNotFound, - InternalMessage: ErrReleaseNotFound, - UserMessage: fmt.Sprintf("no release found with release name '%s'", in.ReleaseName), - } - } return nil, err } return history, nil diff --git a/api/helm-app/applist.pb.go b/api/helm-app/gRPC/applist.pb.go similarity index 99% rename from api/helm-app/applist.pb.go rename to api/helm-app/gRPC/applist.pb.go index c3f9f1f7c1..918a92a433 100644 --- a/api/helm-app/applist.pb.go +++ b/api/helm-app/gRPC/applist.pb.go @@ -4,7 +4,7 @@ // protoc v3.9.1 // source: api/helm-app/applist.proto -package client +package gRPC import ( timestamp "github.com/golang/protobuf/ptypes/timestamp" diff --git a/api/helm-app/applist.proto b/api/helm-app/gRPC/applist.proto similarity index 100% rename from api/helm-app/applist.proto rename to api/helm-app/gRPC/applist.proto diff --git a/api/helm-app/applist_grpc.pb.go b/api/helm-app/gRPC/applist_grpc.pb.go similarity index 99% rename from api/helm-app/applist_grpc.pb.go rename to api/helm-app/gRPC/applist_grpc.pb.go index 4cc1bfa04f..8367a07b69 100644 --- a/api/helm-app/applist_grpc.pb.go +++ b/api/helm-app/gRPC/applist_grpc.pb.go @@ -4,7 +4,7 @@ // - protoc v3.9.1 // source: api/helm-app/applist.proto -package client +package gRPC import ( context "context" diff --git a/api/helm-app/mocks/HelmAppClient.go b/api/helm-app/mocks/HelmAppClient.go index 155cff8d8b..ac80fdd6a9 100644 --- a/api/helm-app/mocks/HelmAppClient.go +++ b/api/helm-app/mocks/HelmAppClient.go @@ -4,9 +4,7 @@ package mocks import ( context "context" - - client "github.com/devtron-labs/devtron/api/helm-app" - + "github.com/devtron-labs/devtron/api/helm-app/gRPC" mock "github.com/stretchr/testify/mock" ) @@ -16,23 +14,23 @@ type HelmAppClient struct { } // DeleteApplication provides a mock function with given fields: ctx, in -func (_m *HelmAppClient) DeleteApplication(ctx context.Context, in *client.ReleaseIdentifier) (*client.UninstallReleaseResponse, error) { +func (_m *HelmAppClient) DeleteApplication(ctx context.Context, in *gRPC.ReleaseIdentifier) (*gRPC.UninstallReleaseResponse, error) { ret := _m.Called(ctx, in) - var r0 *client.UninstallReleaseResponse + var r0 *gRPC.UninstallReleaseResponse var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *client.ReleaseIdentifier) (*client.UninstallReleaseResponse, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.ReleaseIdentifier) (*gRPC.UninstallReleaseResponse, error)); ok { return rf(ctx, in) } - if rf, ok := ret.Get(0).(func(context.Context, *client.ReleaseIdentifier) *client.UninstallReleaseResponse); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.ReleaseIdentifier) *gRPC.UninstallReleaseResponse); ok { r0 = rf(ctx, in) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*client.UninstallReleaseResponse) + r0 = ret.Get(0).(*gRPC.UninstallReleaseResponse) } } - if rf, ok := ret.Get(1).(func(context.Context, *client.ReleaseIdentifier) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, *gRPC.ReleaseIdentifier) error); ok { r1 = rf(ctx, in) } else { r1 = ret.Error(1) @@ -42,23 +40,23 @@ func (_m *HelmAppClient) DeleteApplication(ctx context.Context, in *client.Relea } // GetAppDetail provides a mock function with given fields: ctx, in -func (_m *HelmAppClient) GetAppDetail(ctx context.Context, in *client.AppDetailRequest) (*client.AppDetail, error) { +func (_m *HelmAppClient) GetAppDetail(ctx context.Context, in *gRPC.AppDetailRequest) (*gRPC.AppDetail, error) { ret := _m.Called(ctx, in) - var r0 *client.AppDetail + var r0 *gRPC.AppDetail var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *client.AppDetailRequest) (*client.AppDetail, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.AppDetailRequest) (*gRPC.AppDetail, error)); ok { return rf(ctx, in) } - if rf, ok := ret.Get(0).(func(context.Context, *client.AppDetailRequest) *client.AppDetail); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.AppDetailRequest) *gRPC.AppDetail); ok { r0 = rf(ctx, in) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*client.AppDetail) + r0 = ret.Get(0).(*gRPC.AppDetail) } } - if rf, ok := ret.Get(1).(func(context.Context, *client.AppDetailRequest) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, *gRPC.AppDetailRequest) error); ok { r1 = rf(ctx, in) } else { r1 = ret.Error(1) @@ -68,23 +66,23 @@ func (_m *HelmAppClient) GetAppDetail(ctx context.Context, in *client.AppDetailR } // GetAppStatus provides a mock function with given fields: ctx, in -func (_m *HelmAppClient) GetAppStatus(ctx context.Context, in *client.AppDetailRequest) (*client.AppStatus, error) { +func (_m *HelmAppClient) GetAppStatus(ctx context.Context, in *gRPC.AppDetailRequest) (*gRPC.AppStatus, error) { ret := _m.Called(ctx, in) - var r0 *client.AppStatus + var r0 *gRPC.AppStatus var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *client.AppDetailRequest) (*client.AppStatus, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.AppDetailRequest) (*gRPC.AppStatus, error)); ok { return rf(ctx, in) } - if rf, ok := ret.Get(0).(func(context.Context, *client.AppDetailRequest) *client.AppStatus); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.AppDetailRequest) *gRPC.AppStatus); ok { r0 = rf(ctx, in) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*client.AppStatus) + r0 = ret.Get(0).(*gRPC.AppStatus) } } - if rf, ok := ret.Get(1).(func(context.Context, *client.AppDetailRequest) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, *gRPC.AppDetailRequest) error); ok { r1 = rf(ctx, in) } else { r1 = ret.Error(1) @@ -94,23 +92,23 @@ func (_m *HelmAppClient) GetAppStatus(ctx context.Context, in *client.AppDetailR } // GetDeploymentDetail provides a mock function with given fields: ctx, in -func (_m *HelmAppClient) GetDeploymentDetail(ctx context.Context, in *client.DeploymentDetailRequest) (*client.DeploymentDetailResponse, error) { +func (_m *HelmAppClient) GetDeploymentDetail(ctx context.Context, in *gRPC.DeploymentDetailRequest) (*gRPC.DeploymentDetailResponse, error) { ret := _m.Called(ctx, in) - var r0 *client.DeploymentDetailResponse + var r0 *gRPC.DeploymentDetailResponse var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *client.DeploymentDetailRequest) (*client.DeploymentDetailResponse, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.DeploymentDetailRequest) (*gRPC.DeploymentDetailResponse, error)); ok { return rf(ctx, in) } - if rf, ok := ret.Get(0).(func(context.Context, *client.DeploymentDetailRequest) *client.DeploymentDetailResponse); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.DeploymentDetailRequest) *gRPC.DeploymentDetailResponse); ok { r0 = rf(ctx, in) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*client.DeploymentDetailResponse) + r0 = ret.Get(0).(*gRPC.DeploymentDetailResponse) } } - if rf, ok := ret.Get(1).(func(context.Context, *client.DeploymentDetailRequest) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, *gRPC.DeploymentDetailRequest) error); ok { r1 = rf(ctx, in) } else { r1 = ret.Error(1) @@ -120,23 +118,23 @@ func (_m *HelmAppClient) GetDeploymentDetail(ctx context.Context, in *client.Dep } // GetDeploymentHistory provides a mock function with given fields: ctx, in -func (_m *HelmAppClient) GetDeploymentHistory(ctx context.Context, in *client.AppDetailRequest) (*client.HelmAppDeploymentHistory, error) { +func (_m *HelmAppClient) GetDeploymentHistory(ctx context.Context, in *gRPC.AppDetailRequest) (*gRPC.HelmAppDeploymentHistory, error) { ret := _m.Called(ctx, in) - var r0 *client.HelmAppDeploymentHistory + var r0 *gRPC.HelmAppDeploymentHistory var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *client.AppDetailRequest) (*client.HelmAppDeploymentHistory, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.AppDetailRequest) (*gRPC.HelmAppDeploymentHistory, error)); ok { return rf(ctx, in) } - if rf, ok := ret.Get(0).(func(context.Context, *client.AppDetailRequest) *client.HelmAppDeploymentHistory); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.AppDetailRequest) *gRPC.HelmAppDeploymentHistory); ok { r0 = rf(ctx, in) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*client.HelmAppDeploymentHistory) + r0 = ret.Get(0).(*gRPC.HelmAppDeploymentHistory) } } - if rf, ok := ret.Get(1).(func(context.Context, *client.AppDetailRequest) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, *gRPC.AppDetailRequest) error); ok { r1 = rf(ctx, in) } else { r1 = ret.Error(1) @@ -146,23 +144,23 @@ func (_m *HelmAppClient) GetDeploymentHistory(ctx context.Context, in *client.Ap } // GetDesiredManifest provides a mock function with given fields: ctx, in -func (_m *HelmAppClient) GetDesiredManifest(ctx context.Context, in *client.ObjectRequest) (*client.DesiredManifestResponse, error) { +func (_m *HelmAppClient) GetDesiredManifest(ctx context.Context, in *gRPC.ObjectRequest) (*gRPC.DesiredManifestResponse, error) { ret := _m.Called(ctx, in) - var r0 *client.DesiredManifestResponse + var r0 *gRPC.DesiredManifestResponse var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *client.ObjectRequest) (*client.DesiredManifestResponse, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.ObjectRequest) (*gRPC.DesiredManifestResponse, error)); ok { return rf(ctx, in) } - if rf, ok := ret.Get(0).(func(context.Context, *client.ObjectRequest) *client.DesiredManifestResponse); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.ObjectRequest) *gRPC.DesiredManifestResponse); ok { r0 = rf(ctx, in) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*client.DesiredManifestResponse) + r0 = ret.Get(0).(*gRPC.DesiredManifestResponse) } } - if rf, ok := ret.Get(1).(func(context.Context, *client.ObjectRequest) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, *gRPC.ObjectRequest) error); ok { r1 = rf(ctx, in) } else { r1 = ret.Error(1) @@ -172,23 +170,23 @@ func (_m *HelmAppClient) GetDesiredManifest(ctx context.Context, in *client.Obje } // GetNotes provides a mock function with given fields: ctx, request -func (_m *HelmAppClient) GetNotes(ctx context.Context, request *client.InstallReleaseRequest) (*client.ChartNotesResponse, error) { +func (_m *HelmAppClient) GetNotes(ctx context.Context, request *gRPC.InstallReleaseRequest) (*gRPC.ChartNotesResponse, error) { ret := _m.Called(ctx, request) - var r0 *client.ChartNotesResponse + var r0 *gRPC.ChartNotesResponse var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *client.InstallReleaseRequest) (*client.ChartNotesResponse, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.InstallReleaseRequest) (*gRPC.ChartNotesResponse, error)); ok { return rf(ctx, request) } - if rf, ok := ret.Get(0).(func(context.Context, *client.InstallReleaseRequest) *client.ChartNotesResponse); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.InstallReleaseRequest) *gRPC.ChartNotesResponse); ok { r0 = rf(ctx, request) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*client.ChartNotesResponse) + r0 = ret.Get(0).(*gRPC.ChartNotesResponse) } } - if rf, ok := ret.Get(1).(func(context.Context, *client.InstallReleaseRequest) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, *gRPC.InstallReleaseRequest) error); ok { r1 = rf(ctx, request) } else { r1 = ret.Error(1) @@ -198,23 +196,23 @@ func (_m *HelmAppClient) GetNotes(ctx context.Context, request *client.InstallRe } // GetValuesYaml provides a mock function with given fields: ctx, in -func (_m *HelmAppClient) GetValuesYaml(ctx context.Context, in *client.AppDetailRequest) (*client.ReleaseInfo, error) { +func (_m *HelmAppClient) GetValuesYaml(ctx context.Context, in *gRPC.AppDetailRequest) (*gRPC.ReleaseInfo, error) { ret := _m.Called(ctx, in) - var r0 *client.ReleaseInfo + var r0 *gRPC.ReleaseInfo var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *client.AppDetailRequest) (*client.ReleaseInfo, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.AppDetailRequest) (*gRPC.ReleaseInfo, error)); ok { return rf(ctx, in) } - if rf, ok := ret.Get(0).(func(context.Context, *client.AppDetailRequest) *client.ReleaseInfo); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.AppDetailRequest) *gRPC.ReleaseInfo); ok { r0 = rf(ctx, in) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*client.ReleaseInfo) + r0 = ret.Get(0).(*gRPC.ReleaseInfo) } } - if rf, ok := ret.Get(1).(func(context.Context, *client.AppDetailRequest) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, *gRPC.AppDetailRequest) error); ok { r1 = rf(ctx, in) } else { r1 = ret.Error(1) @@ -224,23 +222,23 @@ func (_m *HelmAppClient) GetValuesYaml(ctx context.Context, in *client.AppDetail } // Hibernate provides a mock function with given fields: ctx, in -func (_m *HelmAppClient) Hibernate(ctx context.Context, in *client.HibernateRequest) (*client.HibernateResponse, error) { +func (_m *HelmAppClient) Hibernate(ctx context.Context, in *gRPC.HibernateRequest) (*gRPC.HibernateResponse, error) { ret := _m.Called(ctx, in) - var r0 *client.HibernateResponse + var r0 *gRPC.HibernateResponse var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *client.HibernateRequest) (*client.HibernateResponse, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.HibernateRequest) (*gRPC.HibernateResponse, error)); ok { return rf(ctx, in) } - if rf, ok := ret.Get(0).(func(context.Context, *client.HibernateRequest) *client.HibernateResponse); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.HibernateRequest) *gRPC.HibernateResponse); ok { r0 = rf(ctx, in) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*client.HibernateResponse) + r0 = ret.Get(0).(*gRPC.HibernateResponse) } } - if rf, ok := ret.Get(1).(func(context.Context, *client.HibernateRequest) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, *gRPC.HibernateRequest) error); ok { r1 = rf(ctx, in) } else { r1 = ret.Error(1) @@ -250,23 +248,23 @@ func (_m *HelmAppClient) Hibernate(ctx context.Context, in *client.HibernateRequ } // InstallRelease provides a mock function with given fields: ctx, in -func (_m *HelmAppClient) InstallRelease(ctx context.Context, in *client.InstallReleaseRequest) (*client.InstallReleaseResponse, error) { +func (_m *HelmAppClient) InstallRelease(ctx context.Context, in *gRPC.InstallReleaseRequest) (*gRPC.InstallReleaseResponse, error) { ret := _m.Called(ctx, in) - var r0 *client.InstallReleaseResponse + var r0 *gRPC.InstallReleaseResponse var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *client.InstallReleaseRequest) (*client.InstallReleaseResponse, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.InstallReleaseRequest) (*gRPC.InstallReleaseResponse, error)); ok { return rf(ctx, in) } - if rf, ok := ret.Get(0).(func(context.Context, *client.InstallReleaseRequest) *client.InstallReleaseResponse); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.InstallReleaseRequest) *gRPC.InstallReleaseResponse); ok { r0 = rf(ctx, in) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*client.InstallReleaseResponse) + r0 = ret.Get(0).(*gRPC.InstallReleaseResponse) } } - if rf, ok := ret.Get(1).(func(context.Context, *client.InstallReleaseRequest) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, *gRPC.InstallReleaseRequest) error); ok { r1 = rf(ctx, in) } else { r1 = ret.Error(1) @@ -276,23 +274,23 @@ func (_m *HelmAppClient) InstallRelease(ctx context.Context, in *client.InstallR } // InstallReleaseWithCustomChart provides a mock function with given fields: ctx, in -func (_m *HelmAppClient) InstallReleaseWithCustomChart(ctx context.Context, in *client.HelmInstallCustomRequest) (*client.HelmInstallCustomResponse, error) { +func (_m *HelmAppClient) InstallReleaseWithCustomChart(ctx context.Context, in *gRPC.HelmInstallCustomRequest) (*gRPC.HelmInstallCustomResponse, error) { ret := _m.Called(ctx, in) - var r0 *client.HelmInstallCustomResponse + var r0 *gRPC.HelmInstallCustomResponse var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *client.HelmInstallCustomRequest) (*client.HelmInstallCustomResponse, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.HelmInstallCustomRequest) (*gRPC.HelmInstallCustomResponse, error)); ok { return rf(ctx, in) } - if rf, ok := ret.Get(0).(func(context.Context, *client.HelmInstallCustomRequest) *client.HelmInstallCustomResponse); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.HelmInstallCustomRequest) *gRPC.HelmInstallCustomResponse); ok { r0 = rf(ctx, in) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*client.HelmInstallCustomResponse) + r0 = ret.Get(0).(*gRPC.HelmInstallCustomResponse) } } - if rf, ok := ret.Get(1).(func(context.Context, *client.HelmInstallCustomRequest) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, *gRPC.HelmInstallCustomRequest) error); ok { r1 = rf(ctx, in) } else { r1 = ret.Error(1) @@ -302,23 +300,23 @@ func (_m *HelmAppClient) InstallReleaseWithCustomChart(ctx context.Context, in * } // IsReleaseInstalled provides a mock function with given fields: ctx, in -func (_m *HelmAppClient) IsReleaseInstalled(ctx context.Context, in *client.ReleaseIdentifier) (*client.BooleanResponse, error) { +func (_m *HelmAppClient) IsReleaseInstalled(ctx context.Context, in *gRPC.ReleaseIdentifier) (*gRPC.BooleanResponse, error) { ret := _m.Called(ctx, in) - var r0 *client.BooleanResponse + var r0 *gRPC.BooleanResponse var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *client.ReleaseIdentifier) (*client.BooleanResponse, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.ReleaseIdentifier) (*gRPC.BooleanResponse, error)); ok { return rf(ctx, in) } - if rf, ok := ret.Get(0).(func(context.Context, *client.ReleaseIdentifier) *client.BooleanResponse); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.ReleaseIdentifier) *gRPC.BooleanResponse); ok { r0 = rf(ctx, in) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*client.BooleanResponse) + r0 = ret.Get(0).(*gRPC.BooleanResponse) } } - if rf, ok := ret.Get(1).(func(context.Context, *client.ReleaseIdentifier) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, *gRPC.ReleaseIdentifier) error); ok { r1 = rf(ctx, in) } else { r1 = ret.Error(1) @@ -328,23 +326,23 @@ func (_m *HelmAppClient) IsReleaseInstalled(ctx context.Context, in *client.Rele } // ListApplication provides a mock function with given fields: ctx, req -func (_m *HelmAppClient) ListApplication(ctx context.Context, req *client.AppListRequest) (client.ApplicationService_ListApplicationsClient, error) { +func (_m *HelmAppClient) ListApplication(ctx context.Context, req *gRPC.AppListRequest) (gRPC.ApplicationService_ListApplicationsClient, error) { ret := _m.Called(ctx, req) - var r0 client.ApplicationService_ListApplicationsClient + var r0 gRPC.ApplicationService_ListApplicationsClient var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *client.AppListRequest) (client.ApplicationService_ListApplicationsClient, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.AppListRequest) (gRPC.ApplicationService_ListApplicationsClient, error)); ok { return rf(ctx, req) } - if rf, ok := ret.Get(0).(func(context.Context, *client.AppListRequest) client.ApplicationService_ListApplicationsClient); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.AppListRequest) gRPC.ApplicationService_ListApplicationsClient); ok { r0 = rf(ctx, req) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(client.ApplicationService_ListApplicationsClient) + r0 = ret.Get(0).(gRPC.ApplicationService_ListApplicationsClient) } } - if rf, ok := ret.Get(1).(func(context.Context, *client.AppListRequest) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, *gRPC.AppListRequest) error); ok { r1 = rf(ctx, req) } else { r1 = ret.Error(1) @@ -354,23 +352,23 @@ func (_m *HelmAppClient) ListApplication(ctx context.Context, req *client.AppLis } // RollbackRelease provides a mock function with given fields: ctx, in -func (_m *HelmAppClient) RollbackRelease(ctx context.Context, in *client.RollbackReleaseRequest) (*client.BooleanResponse, error) { +func (_m *HelmAppClient) RollbackRelease(ctx context.Context, in *gRPC.RollbackReleaseRequest) (*gRPC.BooleanResponse, error) { ret := _m.Called(ctx, in) - var r0 *client.BooleanResponse + var r0 *gRPC.BooleanResponse var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *client.RollbackReleaseRequest) (*client.BooleanResponse, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.RollbackReleaseRequest) (*gRPC.BooleanResponse, error)); ok { return rf(ctx, in) } - if rf, ok := ret.Get(0).(func(context.Context, *client.RollbackReleaseRequest) *client.BooleanResponse); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.RollbackReleaseRequest) *gRPC.BooleanResponse); ok { r0 = rf(ctx, in) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*client.BooleanResponse) + r0 = ret.Get(0).(*gRPC.BooleanResponse) } } - if rf, ok := ret.Get(1).(func(context.Context, *client.RollbackReleaseRequest) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, *gRPC.RollbackReleaseRequest) error); ok { r1 = rf(ctx, in) } else { r1 = ret.Error(1) @@ -380,23 +378,23 @@ func (_m *HelmAppClient) RollbackRelease(ctx context.Context, in *client.Rollbac } // TemplateChart provides a mock function with given fields: ctx, in -func (_m *HelmAppClient) TemplateChart(ctx context.Context, in *client.InstallReleaseRequest) (*client.TemplateChartResponse, error) { +func (_m *HelmAppClient) TemplateChart(ctx context.Context, in *gRPC.InstallReleaseRequest) (*gRPC.TemplateChartResponse, error) { ret := _m.Called(ctx, in) - var r0 *client.TemplateChartResponse + var r0 *gRPC.TemplateChartResponse var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *client.InstallReleaseRequest) (*client.TemplateChartResponse, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.InstallReleaseRequest) (*gRPC.TemplateChartResponse, error)); ok { return rf(ctx, in) } - if rf, ok := ret.Get(0).(func(context.Context, *client.InstallReleaseRequest) *client.TemplateChartResponse); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.InstallReleaseRequest) *gRPC.TemplateChartResponse); ok { r0 = rf(ctx, in) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*client.TemplateChartResponse) + r0 = ret.Get(0).(*gRPC.TemplateChartResponse) } } - if rf, ok := ret.Get(1).(func(context.Context, *client.InstallReleaseRequest) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, *gRPC.InstallReleaseRequest) error); ok { r1 = rf(ctx, in) } else { r1 = ret.Error(1) @@ -406,23 +404,23 @@ func (_m *HelmAppClient) TemplateChart(ctx context.Context, in *client.InstallRe } // UnHibernate provides a mock function with given fields: ctx, in -func (_m *HelmAppClient) UnHibernate(ctx context.Context, in *client.HibernateRequest) (*client.HibernateResponse, error) { +func (_m *HelmAppClient) UnHibernate(ctx context.Context, in *gRPC.HibernateRequest) (*gRPC.HibernateResponse, error) { ret := _m.Called(ctx, in) - var r0 *client.HibernateResponse + var r0 *gRPC.HibernateResponse var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *client.HibernateRequest) (*client.HibernateResponse, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.HibernateRequest) (*gRPC.HibernateResponse, error)); ok { return rf(ctx, in) } - if rf, ok := ret.Get(0).(func(context.Context, *client.HibernateRequest) *client.HibernateResponse); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.HibernateRequest) *gRPC.HibernateResponse); ok { r0 = rf(ctx, in) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*client.HibernateResponse) + r0 = ret.Get(0).(*gRPC.HibernateResponse) } } - if rf, ok := ret.Get(1).(func(context.Context, *client.HibernateRequest) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, *gRPC.HibernateRequest) error); ok { r1 = rf(ctx, in) } else { r1 = ret.Error(1) @@ -432,23 +430,23 @@ func (_m *HelmAppClient) UnHibernate(ctx context.Context, in *client.HibernateRe } // UpdateApplication provides a mock function with given fields: ctx, in -func (_m *HelmAppClient) UpdateApplication(ctx context.Context, in *client.UpgradeReleaseRequest) (*client.UpgradeReleaseResponse, error) { +func (_m *HelmAppClient) UpdateApplication(ctx context.Context, in *gRPC.UpgradeReleaseRequest) (*gRPC.UpgradeReleaseResponse, error) { ret := _m.Called(ctx, in) - var r0 *client.UpgradeReleaseResponse + var r0 *gRPC.UpgradeReleaseResponse var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *client.UpgradeReleaseRequest) (*client.UpgradeReleaseResponse, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.UpgradeReleaseRequest) (*gRPC.UpgradeReleaseResponse, error)); ok { return rf(ctx, in) } - if rf, ok := ret.Get(0).(func(context.Context, *client.UpgradeReleaseRequest) *client.UpgradeReleaseResponse); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.UpgradeReleaseRequest) *gRPC.UpgradeReleaseResponse); ok { r0 = rf(ctx, in) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*client.UpgradeReleaseResponse) + r0 = ret.Get(0).(*gRPC.UpgradeReleaseResponse) } } - if rf, ok := ret.Get(1).(func(context.Context, *client.UpgradeReleaseRequest) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, *gRPC.UpgradeReleaseRequest) error); ok { r1 = rf(ctx, in) } else { r1 = ret.Error(1) @@ -458,23 +456,23 @@ func (_m *HelmAppClient) UpdateApplication(ctx context.Context, in *client.Upgra } // UpdateApplicationWithChartInfo provides a mock function with given fields: ctx, in -func (_m *HelmAppClient) UpdateApplicationWithChartInfo(ctx context.Context, in *client.InstallReleaseRequest) (*client.UpgradeReleaseResponse, error) { +func (_m *HelmAppClient) UpdateApplicationWithChartInfo(ctx context.Context, in *gRPC.InstallReleaseRequest) (*gRPC.UpgradeReleaseResponse, error) { ret := _m.Called(ctx, in) - var r0 *client.UpgradeReleaseResponse + var r0 *gRPC.UpgradeReleaseResponse var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *client.InstallReleaseRequest) (*client.UpgradeReleaseResponse, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.InstallReleaseRequest) (*gRPC.UpgradeReleaseResponse, error)); ok { return rf(ctx, in) } - if rf, ok := ret.Get(0).(func(context.Context, *client.InstallReleaseRequest) *client.UpgradeReleaseResponse); ok { + if rf, ok := ret.Get(0).(func(context.Context, *gRPC.InstallReleaseRequest) *gRPC.UpgradeReleaseResponse); ok { r0 = rf(ctx, in) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*client.UpgradeReleaseResponse) + r0 = ret.Get(0).(*gRPC.UpgradeReleaseResponse) } } - if rf, ok := ret.Get(1).(func(context.Context, *client.InstallReleaseRequest) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, *gRPC.InstallReleaseRequest) error); ok { r1 = rf(ctx, in) } else { r1 = ret.Error(1) diff --git a/api/helm-app/mocks/HelmAppService.go b/api/helm-app/mocks/HelmAppService.go index 59dce9f450..895130dd23 100644 --- a/api/helm-app/mocks/HelmAppService.go +++ b/api/helm-app/mocks/HelmAppService.go @@ -4,8 +4,9 @@ package mocks import ( context "context" - - client "github.com/devtron-labs/devtron/api/helm-app" + "github.com/devtron-labs/devtron/api/helm-app/bean" + bean2 "github.com/devtron-labs/devtron/api/helm-app/gRPC" + client "github.com/devtron-labs/devtron/api/helm-app/service" http "net/http" @@ -88,19 +89,19 @@ func (_m *HelmAppService) EncodeAppId(appIdentifier *client.AppIdentifier) strin } // GetApplicationDetail provides a mock function with given fields: ctx, app -func (_m *HelmAppService) GetApplicationDetail(ctx context.Context, app *client.AppIdentifier) (*client.AppDetail, error) { +func (_m *HelmAppService) GetApplicationDetail(ctx context.Context, app *client.AppIdentifier) (*bean2.AppDetail, error) { ret := _m.Called(ctx, app) - var r0 *client.AppDetail + var r0 *bean2.AppDetail var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *client.AppIdentifier) (*client.AppDetail, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, *client.AppIdentifier) (*bean2.AppDetail, error)); ok { return rf(ctx, app) } - if rf, ok := ret.Get(0).(func(context.Context, *client.AppIdentifier) *client.AppDetail); ok { + if rf, ok := ret.Get(0).(func(context.Context, *client.AppIdentifier) *bean2.AppDetail); ok { r0 = rf(ctx, app) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*client.AppDetail) + r0 = ret.Get(0).(*bean2.AppDetail) } } @@ -114,23 +115,23 @@ func (_m *HelmAppService) GetApplicationDetail(ctx context.Context, app *client. } // GetApplicationDetailWithFilter provides a mock function with given fields: ctx, app, resourceTreeFilter -func (_m *HelmAppService) GetApplicationDetailWithFilter(ctx context.Context, app *client.AppIdentifier, resourceTreeFilter *client.ResourceTreeFilter) (*client.AppDetail, error) { +func (_m *HelmAppService) GetApplicationDetailWithFilter(ctx context.Context, app *client.AppIdentifier, resourceTreeFilter *bean2.ResourceTreeFilter) (*bean2.AppDetail, error) { ret := _m.Called(ctx, app, resourceTreeFilter) - var r0 *client.AppDetail + var r0 *bean2.AppDetail var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *client.AppIdentifier, *client.ResourceTreeFilter) (*client.AppDetail, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, *client.AppIdentifier, *bean2.ResourceTreeFilter) (*bean2.AppDetail, error)); ok { return rf(ctx, app, resourceTreeFilter) } - if rf, ok := ret.Get(0).(func(context.Context, *client.AppIdentifier, *client.ResourceTreeFilter) *client.AppDetail); ok { + if rf, ok := ret.Get(0).(func(context.Context, *client.AppIdentifier, *bean2.ResourceTreeFilter) *bean2.AppDetail); ok { r0 = rf(ctx, app, resourceTreeFilter) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*client.AppDetail) + r0 = ret.Get(0).(*bean2.AppDetail) } } - if rf, ok := ret.Get(1).(func(context.Context, *client.AppIdentifier, *client.ResourceTreeFilter) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, *client.AppIdentifier, *bean2.ResourceTreeFilter) error); ok { r1 = rf(ctx, app, resourceTreeFilter) } else { r1 = ret.Error(1) @@ -164,19 +165,19 @@ func (_m *HelmAppService) GetApplicationStatus(ctx context.Context, app *client. } // GetClusterConf provides a mock function with given fields: clusterId -func (_m *HelmAppService) GetClusterConf(clusterId int) (*client.ClusterConfig, error) { +func (_m *HelmAppService) GetClusterConf(clusterId int) (*bean2.ClusterConfig, error) { ret := _m.Called(clusterId) - var r0 *client.ClusterConfig + var r0 *bean2.ClusterConfig var r1 error - if rf, ok := ret.Get(0).(func(int) (*client.ClusterConfig, error)); ok { + if rf, ok := ret.Get(0).(func(int) (*bean2.ClusterConfig, error)); ok { return rf(clusterId) } - if rf, ok := ret.Get(0).(func(int) *client.ClusterConfig); ok { + if rf, ok := ret.Get(0).(func(int) *bean2.ClusterConfig); ok { r0 = rf(clusterId) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*client.ClusterConfig) + r0 = ret.Get(0).(*bean2.ClusterConfig) } } @@ -216,19 +217,19 @@ func (_m *HelmAppService) GetDeploymentDetail(ctx context.Context, app *client.A } // GetDeploymentHistory provides a mock function with given fields: ctx, app -func (_m *HelmAppService) GetDeploymentHistory(ctx context.Context, app *client.AppIdentifier) (*client.HelmAppDeploymentHistory, error) { +func (_m *HelmAppService) GetDeploymentHistory(ctx context.Context, app *client.AppIdentifier) (*bean2.HelmAppDeploymentHistory, error) { ret := _m.Called(ctx, app) - var r0 *client.HelmAppDeploymentHistory + var r0 *bean2.HelmAppDeploymentHistory var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *client.AppIdentifier) (*client.HelmAppDeploymentHistory, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, *client.AppIdentifier) (*bean2.HelmAppDeploymentHistory, error)); ok { return rf(ctx, app) } - if rf, ok := ret.Get(0).(func(context.Context, *client.AppIdentifier) *client.HelmAppDeploymentHistory); ok { + if rf, ok := ret.Get(0).(func(context.Context, *client.AppIdentifier) *bean2.HelmAppDeploymentHistory); ok { r0 = rf(ctx, app) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*client.HelmAppDeploymentHistory) + r0 = ret.Get(0).(*bean2.HelmAppDeploymentHistory) } } @@ -284,21 +285,21 @@ func (_m *HelmAppService) GetDevtronHelmAppIdentifier() *client.AppIdentifier { } // GetNotes provides a mock function with given fields: ctx, request -func (_m *HelmAppService) GetNotes(ctx context.Context, request *client.InstallReleaseRequest) (string, error) { +func (_m *HelmAppService) GetNotes(ctx context.Context, request *bean2.InstallReleaseRequest) (string, error) { ret := _m.Called(ctx, request) var r0 string var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *client.InstallReleaseRequest) (string, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, *bean2.InstallReleaseRequest) (string, error)); ok { return rf(ctx, request) } - if rf, ok := ret.Get(0).(func(context.Context, *client.InstallReleaseRequest) string); ok { + if rf, ok := ret.Get(0).(func(context.Context, *bean2.InstallReleaseRequest) string); ok { r0 = rf(ctx, request) } else { r0 = ret.Get(0).(string) } - if rf, ok := ret.Get(1).(func(context.Context, *client.InstallReleaseRequest) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, *bean2.InstallReleaseRequest) error); ok { r1 = rf(ctx, request) } else { r1 = ret.Error(1) @@ -308,11 +309,11 @@ func (_m *HelmAppService) GetNotes(ctx context.Context, request *client.InstallR } // GetRevisionHistoryMaxValue provides a mock function with given fields: appType -func (_m *HelmAppService) GetRevisionHistoryMaxValue(appType client.SourceAppType) int32 { +func (_m *HelmAppService) GetRevisionHistoryMaxValue(appType bean.SourceAppType) int32 { ret := _m.Called(appType) var r0 int32 - if rf, ok := ret.Get(0).(func(client.SourceAppType) int32); ok { + if rf, ok := ret.Get(0).(func(bean.SourceAppType) int32); ok { r0 = rf(appType) } else { r0 = ret.Get(0).(int32) @@ -322,19 +323,19 @@ func (_m *HelmAppService) GetRevisionHistoryMaxValue(appType client.SourceAppTyp } // GetValuesYaml provides a mock function with given fields: ctx, app -func (_m *HelmAppService) GetValuesYaml(ctx context.Context, app *client.AppIdentifier) (*client.ReleaseInfo, error) { +func (_m *HelmAppService) GetValuesYaml(ctx context.Context, app *client.AppIdentifier) (*bean2.ReleaseInfo, error) { ret := _m.Called(ctx, app) - var r0 *client.ReleaseInfo + var r0 *bean2.ReleaseInfo var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *client.AppIdentifier) (*client.ReleaseInfo, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, *client.AppIdentifier) (*bean2.ReleaseInfo, error)); ok { return rf(ctx, app) } - if rf, ok := ret.Get(0).(func(context.Context, *client.AppIdentifier) *client.ReleaseInfo); ok { + if rf, ok := ret.Get(0).(func(context.Context, *client.AppIdentifier) *bean2.ReleaseInfo); ok { r0 = rf(ctx, app) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*client.ReleaseInfo) + r0 = ret.Get(0).(*bean2.ReleaseInfo) } } @@ -374,23 +375,23 @@ func (_m *HelmAppService) HibernateApplication(ctx context.Context, app *client. } // InstallRelease provides a mock function with given fields: ctx, clusterId, installReleaseRequest -func (_m *HelmAppService) InstallRelease(ctx context.Context, clusterId int, installReleaseRequest *client.InstallReleaseRequest) (*client.InstallReleaseResponse, error) { +func (_m *HelmAppService) InstallRelease(ctx context.Context, clusterId int, installReleaseRequest *bean2.InstallReleaseRequest) (*bean2.InstallReleaseResponse, error) { ret := _m.Called(ctx, clusterId, installReleaseRequest) - var r0 *client.InstallReleaseResponse + var r0 *bean2.InstallReleaseResponse var r1 error - if rf, ok := ret.Get(0).(func(context.Context, int, *client.InstallReleaseRequest) (*client.InstallReleaseResponse, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, int, *bean2.InstallReleaseRequest) (*bean2.InstallReleaseResponse, error)); ok { return rf(ctx, clusterId, installReleaseRequest) } - if rf, ok := ret.Get(0).(func(context.Context, int, *client.InstallReleaseRequest) *client.InstallReleaseResponse); ok { + if rf, ok := ret.Get(0).(func(context.Context, int, *bean2.InstallReleaseRequest) *bean2.InstallReleaseResponse); ok { r0 = rf(ctx, clusterId, installReleaseRequest) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*client.InstallReleaseResponse) + r0 = ret.Get(0).(*bean2.InstallReleaseResponse) } } - if rf, ok := ret.Get(1).(func(context.Context, int, *client.InstallReleaseRequest) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, int, *bean2.InstallReleaseRequest) error); ok { r1 = rf(ctx, clusterId, installReleaseRequest) } else { r1 = ret.Error(1) @@ -505,15 +506,15 @@ func (_m *HelmAppService) UnHibernateApplication(ctx context.Context, app *clien } // UpdateApplication provides a mock function with given fields: ctx, app, request -func (_m *HelmAppService) UpdateApplication(ctx context.Context, app *client.AppIdentifier, request *client.UpdateApplicationRequestDto) (*openapi.UpdateReleaseResponse, error) { +func (_m *HelmAppService) UpdateApplication(ctx context.Context, app *client.AppIdentifier, request *bean.UpdateApplicationRequestDto) (*openapi.UpdateReleaseResponse, error) { ret := _m.Called(ctx, app, request) var r0 *openapi.UpdateReleaseResponse var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *client.AppIdentifier, *client.UpdateApplicationRequestDto) (*openapi.UpdateReleaseResponse, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, *client.AppIdentifier, *bean.UpdateApplicationRequestDto) (*openapi.UpdateReleaseResponse, error)); ok { return rf(ctx, app, request) } - if rf, ok := ret.Get(0).(func(context.Context, *client.AppIdentifier, *client.UpdateApplicationRequestDto) *openapi.UpdateReleaseResponse); ok { + if rf, ok := ret.Get(0).(func(context.Context, *client.AppIdentifier, *bean.UpdateApplicationRequestDto) *openapi.UpdateReleaseResponse); ok { r0 = rf(ctx, app, request) } else { if ret.Get(0) != nil { @@ -521,7 +522,7 @@ func (_m *HelmAppService) UpdateApplication(ctx context.Context, app *client.App } } - if rf, ok := ret.Get(1).(func(context.Context, *client.AppIdentifier, *client.UpdateApplicationRequestDto) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, *client.AppIdentifier, *bean.UpdateApplicationRequestDto) error); ok { r1 = rf(ctx, app, request) } else { r1 = ret.Error(1) @@ -531,15 +532,15 @@ func (_m *HelmAppService) UpdateApplication(ctx context.Context, app *client.App } // UpdateApplicationWithChartInfo provides a mock function with given fields: ctx, clusterId, request -func (_m *HelmAppService) UpdateApplicationWithChartInfo(ctx context.Context, clusterId int, request *client.UpdateApplicationWithChartInfoRequestDto) (*openapi.UpdateReleaseResponse, error) { +func (_m *HelmAppService) UpdateApplicationWithChartInfo(ctx context.Context, clusterId int, request *bean.UpdateApplicationWithChartInfoRequestDto) (*openapi.UpdateReleaseResponse, error) { ret := _m.Called(ctx, clusterId, request) var r0 *openapi.UpdateReleaseResponse var r1 error - if rf, ok := ret.Get(0).(func(context.Context, int, *client.UpdateApplicationWithChartInfoRequestDto) (*openapi.UpdateReleaseResponse, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, int, *bean.UpdateApplicationWithChartInfoRequestDto) (*openapi.UpdateReleaseResponse, error)); ok { return rf(ctx, clusterId, request) } - if rf, ok := ret.Get(0).(func(context.Context, int, *client.UpdateApplicationWithChartInfoRequestDto) *openapi.UpdateReleaseResponse); ok { + if rf, ok := ret.Get(0).(func(context.Context, int, *bean.UpdateApplicationWithChartInfoRequestDto) *openapi.UpdateReleaseResponse); ok { r0 = rf(ctx, clusterId, request) } else { if ret.Get(0) != nil { @@ -547,7 +548,7 @@ func (_m *HelmAppService) UpdateApplicationWithChartInfo(ctx context.Context, cl } } - if rf, ok := ret.Get(1).(func(context.Context, int, *client.UpdateApplicationWithChartInfoRequestDto) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, int, *bean.UpdateApplicationWithChartInfoRequestDto) error); ok { r1 = rf(ctx, clusterId, request) } else { r1 = ret.Error(1) @@ -557,15 +558,15 @@ func (_m *HelmAppService) UpdateApplicationWithChartInfo(ctx context.Context, cl } // UpdateApplicationWithChartInfoWithExtraValues provides a mock function with given fields: ctx, appIdentifier, chartRepository, extraValues, extraValuesYamlUrl, useLatestChartVersion -func (_m *HelmAppService) UpdateApplicationWithChartInfoWithExtraValues(ctx context.Context, appIdentifier *client.AppIdentifier, chartRepository *client.ChartRepository, extraValues map[string]interface{}, extraValuesYamlUrl string, useLatestChartVersion bool) (*openapi.UpdateReleaseResponse, error) { +func (_m *HelmAppService) UpdateApplicationWithChartInfoWithExtraValues(ctx context.Context, appIdentifier *client.AppIdentifier, chartRepository *bean2.ChartRepository, extraValues map[string]interface{}, extraValuesYamlUrl string, useLatestChartVersion bool) (*openapi.UpdateReleaseResponse, error) { ret := _m.Called(ctx, appIdentifier, chartRepository, extraValues, extraValuesYamlUrl, useLatestChartVersion) var r0 *openapi.UpdateReleaseResponse var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *client.AppIdentifier, *client.ChartRepository, map[string]interface{}, string, bool) (*openapi.UpdateReleaseResponse, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, *client.AppIdentifier, *bean2.ChartRepository, map[string]interface{}, string, bool) (*openapi.UpdateReleaseResponse, error)); ok { return rf(ctx, appIdentifier, chartRepository, extraValues, extraValuesYamlUrl, useLatestChartVersion) } - if rf, ok := ret.Get(0).(func(context.Context, *client.AppIdentifier, *client.ChartRepository, map[string]interface{}, string, bool) *openapi.UpdateReleaseResponse); ok { + if rf, ok := ret.Get(0).(func(context.Context, *client.AppIdentifier, *bean2.ChartRepository, map[string]interface{}, string, bool) *openapi.UpdateReleaseResponse); ok { r0 = rf(ctx, appIdentifier, chartRepository, extraValues, extraValuesYamlUrl, useLatestChartVersion) } else { if ret.Get(0) != nil { @@ -573,7 +574,7 @@ func (_m *HelmAppService) UpdateApplicationWithChartInfoWithExtraValues(ctx cont } } - if rf, ok := ret.Get(1).(func(context.Context, *client.AppIdentifier, *client.ChartRepository, map[string]interface{}, string, bool) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, *client.AppIdentifier, *bean2.ChartRepository, map[string]interface{}, string, bool) error); ok { r1 = rf(ctx, appIdentifier, chartRepository, extraValues, extraValuesYamlUrl, useLatestChartVersion) } else { r1 = ret.Error(1) @@ -583,11 +584,11 @@ func (_m *HelmAppService) UpdateApplicationWithChartInfoWithExtraValues(ctx cont } // ValidateOCIRegistry provides a mock function with given fields: ctx, OCIRegistryRequest -func (_m *HelmAppService) ValidateOCIRegistry(ctx context.Context, OCIRegistryRequest *client.RegistryCredential) bool { +func (_m *HelmAppService) ValidateOCIRegistry(ctx context.Context, OCIRegistryRequest *bean2.RegistryCredential) bool { ret := _m.Called(ctx, OCIRegistryRequest) var r0 bool - if rf, ok := ret.Get(0).(func(context.Context, *client.RegistryCredential) bool); ok { + if rf, ok := ret.Get(0).(func(context.Context, *bean2.RegistryCredential) bool); ok { r0 = rf(ctx, OCIRegistryRequest) } else { r0 = ret.Get(0).(bool) diff --git a/api/helm-app/HelmAppService.go b/api/helm-app/service/HelmAppService.go similarity index 89% rename from api/helm-app/HelmAppService.go rename to api/helm-app/service/HelmAppService.go index 8485431c26..797c28e374 100644 --- a/api/helm-app/HelmAppService.go +++ b/api/helm-app/service/HelmAppService.go @@ -1,11 +1,14 @@ -package client +package service import ( "context" "errors" "fmt" "github.com/devtron-labs/common-lib/utils/k8s" + "github.com/devtron-labs/devtron/api/helm-app/bean" + "github.com/devtron-labs/devtron/api/helm-app/gRPC" "github.com/devtron-labs/devtron/api/helm-app/models" + "github.com/devtron-labs/devtron/internal/constants" repository2 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" "github.com/go-pg/pg" "google.golang.org/grpc/codes" @@ -43,37 +46,37 @@ import ( type HelmAppService interface { ListHelmApplications(ctx context.Context, clusterIds []int, w http.ResponseWriter, token string, helmAuth func(token string, object string) bool) - GetApplicationDetail(ctx context.Context, app *AppIdentifier) (*AppDetail, error) - GetApplicationAndReleaseStatus(ctx context.Context, app *AppIdentifier) (*AppStatus, error) - GetApplicationDetailWithFilter(ctx context.Context, app *AppIdentifier, resourceTreeFilter *ResourceTreeFilter) (*AppDetail, error) + GetApplicationDetail(ctx context.Context, app *AppIdentifier) (*gRPC.AppDetail, error) + GetApplicationAndReleaseStatus(ctx context.Context, app *AppIdentifier) (*gRPC.AppStatus, error) + GetApplicationDetailWithFilter(ctx context.Context, app *AppIdentifier, resourceTreeFilter *gRPC.ResourceTreeFilter) (*gRPC.AppDetail, error) HibernateApplication(ctx context.Context, app *AppIdentifier, hibernateRequest *openapi.HibernateRequest) ([]*openapi.HibernateStatus, error) UnHibernateApplication(ctx context.Context, app *AppIdentifier, hibernateRequest *openapi.HibernateRequest) ([]*openapi.HibernateStatus, error) DecodeAppId(appId string) (*AppIdentifier, error) EncodeAppId(appIdentifier *AppIdentifier) string - GetDeploymentHistory(ctx context.Context, app *AppIdentifier) (*HelmAppDeploymentHistory, error) - GetValuesYaml(ctx context.Context, app *AppIdentifier) (*ReleaseInfo, error) + GetDeploymentHistory(ctx context.Context, app *AppIdentifier) (*gRPC.HelmAppDeploymentHistory, error) + GetValuesYaml(ctx context.Context, app *AppIdentifier) (*gRPC.ReleaseInfo, error) GetDesiredManifest(ctx context.Context, app *AppIdentifier, resource *openapi.ResourceIdentifier) (*openapi.DesiredManifestResponse, error) DeleteApplication(ctx context.Context, app *AppIdentifier) (*openapi.UninstallReleaseResponse, error) DeleteDBLinkedHelmApplication(ctx context.Context, app *AppIdentifier, useId int32) (*openapi.UninstallReleaseResponse, error) - UpdateApplication(ctx context.Context, app *AppIdentifier, request *UpdateApplicationRequestDto) (*openapi.UpdateReleaseResponse, error) + UpdateApplication(ctx context.Context, app *AppIdentifier, request *bean.UpdateApplicationRequestDto) (*openapi.UpdateReleaseResponse, error) GetDeploymentDetail(ctx context.Context, app *AppIdentifier, version int32) (*openapi.HelmAppDeploymentManifestDetail, error) - InstallRelease(ctx context.Context, clusterId int, installReleaseRequest *InstallReleaseRequest) (*InstallReleaseResponse, error) - UpdateApplicationWithChartInfo(ctx context.Context, clusterId int, request *UpdateApplicationWithChartInfoRequestDto) (*openapi.UpdateReleaseResponse, error) + InstallRelease(ctx context.Context, clusterId int, installReleaseRequest *gRPC.InstallReleaseRequest) (*gRPC.InstallReleaseResponse, error) + UpdateApplicationWithChartInfo(ctx context.Context, clusterId int, request *bean.UpdateApplicationWithChartInfoRequestDto) (*openapi.UpdateReleaseResponse, error) IsReleaseInstalled(ctx context.Context, app *AppIdentifier) (bool, error) RollbackRelease(ctx context.Context, app *AppIdentifier, version int32) (bool, error) - GetClusterConf(clusterId int) (*ClusterConfig, error) + GetClusterConf(clusterId int) (*gRPC.ClusterConfig, error) GetDevtronHelmAppIdentifier() *AppIdentifier - UpdateApplicationWithChartInfoWithExtraValues(ctx context.Context, appIdentifier *AppIdentifier, chartRepository *ChartRepository, extraValues map[string]interface{}, extraValuesYamlUrl string, useLatestChartVersion bool) (*openapi.UpdateReleaseResponse, error) + UpdateApplicationWithChartInfoWithExtraValues(ctx context.Context, appIdentifier *AppIdentifier, chartRepository *gRPC.ChartRepository, extraValues map[string]interface{}, extraValuesYamlUrl string, useLatestChartVersion bool) (*openapi.UpdateReleaseResponse, error) TemplateChart(ctx context.Context, templateChartRequest *openapi2.TemplateChartRequest) (*openapi2.TemplateChartResponse, error) - GetNotes(ctx context.Context, request *InstallReleaseRequest) (string, error) - ValidateOCIRegistry(ctx context.Context, OCIRegistryRequest *RegistryCredential) bool - GetRevisionHistoryMaxValue(appType SourceAppType) int32 + GetNotes(ctx context.Context, request *gRPC.InstallReleaseRequest) (string, error) + ValidateOCIRegistry(ctx context.Context, OCIRegistryRequest *gRPC.RegistryCredential) bool + GetRevisionHistoryMaxValue(appType bean.SourceAppType) int32 } type HelmAppServiceImpl struct { logger *zap.SugaredLogger clusterService cluster.ClusterService - helmAppClient HelmAppClient + helmAppClient gRPC.HelmAppClient pump connector.Pump enforcerUtil rbac.EnforcerUtilHelm serverDataStore *serverDataStore.ServerDataStore @@ -89,7 +92,7 @@ type HelmAppServiceImpl struct { } func NewHelmAppServiceImpl(Logger *zap.SugaredLogger, clusterService cluster.ClusterService, - helmAppClient HelmAppClient, pump connector.Pump, enforcerUtil rbac.EnforcerUtilHelm, + helmAppClient gRPC.HelmAppClient, pump connector.Pump, enforcerUtil rbac.EnforcerUtilHelm, serverDataStore *serverDataStore.ServerDataStore, serverEnvConfig *serverEnvConfig.ServerEnvConfig, appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository, environmentService cluster.EnvironmentService, pipelineRepository pipelineConfig.PipelineRepository, @@ -127,7 +130,7 @@ func GetHelmReleaseConfig() (*HelmReleaseConfig, error) { return cfg, err } -func (impl *HelmAppServiceImpl) listApplications(ctx context.Context, clusterIds []int) (ApplicationService_ListApplicationsClient, error) { +func (impl *HelmAppServiceImpl) listApplications(ctx context.Context, clusterIds []int) (gRPC.ApplicationService_ListApplicationsClient, error) { if len(clusterIds) == 0 { return nil, nil } @@ -138,9 +141,9 @@ func (impl *HelmAppServiceImpl) listApplications(ctx context.Context, clusterIds impl.logger.Errorw("error in fetching cluster detail", "err", err) return nil, err } - req := &AppListRequest{} + req := &gRPC.AppListRequest{} for _, clusterDetail := range clusters { - config := &ClusterConfig{ + config := &gRPC.ClusterConfig{ ApiServerUrl: clusterDetail.ServerUrl, Token: clusterDetail.Config[k8s.BearerToken], ClusterId: int32(clusterDetail.Id), @@ -198,14 +201,14 @@ func (impl *HelmAppServiceImpl) ListHelmApplications(ctx context.Context, cluste return appStream.Recv() }, err, func(message interface{}) interface{} { - return impl.appListRespProtoTransformer(message.(*DeployedAppList), token, helmAuth, helmCdPipelines, installedHelmApps) + return impl.appListRespProtoTransformer(message.(*gRPC.DeployedAppList), token, helmAuth, helmCdPipelines, installedHelmApps) }) } -func (impl *HelmAppServiceImpl) hibernateReqAdaptor(hibernateRequest *openapi.HibernateRequest) *HibernateRequest { - req := &HibernateRequest{} +func (impl *HelmAppServiceImpl) hibernateReqAdaptor(hibernateRequest *openapi.HibernateRequest) *gRPC.HibernateRequest { + req := &gRPC.HibernateRequest{} for _, reqObject := range hibernateRequest.GetResources() { - obj := &ObjectIdentifier{ + obj := &gRPC.ObjectIdentifier{ Group: *reqObject.Group, Kind: *reqObject.Kind, Version: *reqObject.Version, @@ -216,7 +219,7 @@ func (impl *HelmAppServiceImpl) hibernateReqAdaptor(hibernateRequest *openapi.Hi } return req } -func (impl *HelmAppServiceImpl) hibernateResponseAdaptor(in []*HibernateStatus) []*openapi.HibernateStatus { +func (impl *HelmAppServiceImpl) hibernateResponseAdaptor(in []*gRPC.HibernateStatus) []*openapi.HibernateStatus { var resStatus []*openapi.HibernateStatus for _, status := range in { resObj := &openapi.HibernateStatus{ @@ -265,13 +268,13 @@ func (impl *HelmAppServiceImpl) UnHibernateApplication(ctx context.Context, app return response, nil } -func (impl *HelmAppServiceImpl) GetClusterConf(clusterId int) (*ClusterConfig, error) { +func (impl *HelmAppServiceImpl) GetClusterConf(clusterId int) (*gRPC.ClusterConfig, error) { cluster, err := impl.clusterService.FindById(clusterId) if err != nil { impl.logger.Errorw("error in fetching cluster detail", "err", err) return nil, err } - config := &ClusterConfig{ + config := &gRPC.ClusterConfig{ ApiServerUrl: cluster.ServerUrl, Token: cluster.Config[k8s.BearerToken], ClusterId: int32(cluster.Id), @@ -286,25 +289,25 @@ func (impl *HelmAppServiceImpl) GetClusterConf(clusterId int) (*ClusterConfig, e return config, nil } -func (impl *HelmAppServiceImpl) GetApplicationDetail(ctx context.Context, app *AppIdentifier) (*AppDetail, error) { +func (impl *HelmAppServiceImpl) GetApplicationDetail(ctx context.Context, app *AppIdentifier) (*gRPC.AppDetail, error) { return impl.getApplicationDetail(ctx, app, nil) } -func (impl *HelmAppServiceImpl) GetApplicationAndReleaseStatus(ctx context.Context, app *AppIdentifier) (*AppStatus, error) { +func (impl *HelmAppServiceImpl) GetApplicationAndReleaseStatus(ctx context.Context, app *AppIdentifier) (*gRPC.AppStatus, error) { return impl.getApplicationAndReleaseStatus(ctx, app) } -func (impl *HelmAppServiceImpl) GetApplicationDetailWithFilter(ctx context.Context, app *AppIdentifier, resourceTreeFilter *ResourceTreeFilter) (*AppDetail, error) { +func (impl *HelmAppServiceImpl) GetApplicationDetailWithFilter(ctx context.Context, app *AppIdentifier, resourceTreeFilter *gRPC.ResourceTreeFilter) (*gRPC.AppDetail, error) { return impl.getApplicationDetail(ctx, app, resourceTreeFilter) } -func (impl *HelmAppServiceImpl) getApplicationDetail(ctx context.Context, app *AppIdentifier, resourceTreeFilter *ResourceTreeFilter) (*AppDetail, error) { +func (impl *HelmAppServiceImpl) getApplicationDetail(ctx context.Context, app *AppIdentifier, resourceTreeFilter *gRPC.ResourceTreeFilter) (*gRPC.AppDetail, error) { config, err := impl.GetClusterConf(app.ClusterId) if err != nil { impl.logger.Errorw("error in fetching cluster detail", "err", err) return nil, err } - req := &AppDetailRequest{ + req := &gRPC.AppDetailRequest{ ClusterConfig: config, Namespace: app.Namespace, ReleaseName: app.ReleaseName, @@ -334,13 +337,13 @@ func (impl *HelmAppServiceImpl) getApplicationDetail(ctx context.Context, app *A return appdetail, err } -func (impl *HelmAppServiceImpl) getApplicationAndReleaseStatus(ctx context.Context, app *AppIdentifier) (*AppStatus, error) { +func (impl *HelmAppServiceImpl) getApplicationAndReleaseStatus(ctx context.Context, app *AppIdentifier) (*gRPC.AppStatus, error) { config, err := impl.GetClusterConf(app.ClusterId) if err != nil { impl.logger.Errorw("error in fetching cluster detail", "err", err) return nil, err } - req := &AppDetailRequest{ + req := &gRPC.AppDetailRequest{ ClusterConfig: config, Namespace: app.Namespace, ReleaseName: app.ReleaseName, @@ -353,28 +356,35 @@ func (impl *HelmAppServiceImpl) getApplicationAndReleaseStatus(ctx context.Conte return appStatus, err } -func (impl *HelmAppServiceImpl) GetDeploymentHistory(ctx context.Context, app *AppIdentifier) (*HelmAppDeploymentHistory, error) { +func (impl *HelmAppServiceImpl) GetDeploymentHistory(ctx context.Context, app *AppIdentifier) (*gRPC.HelmAppDeploymentHistory, error) { config, err := impl.GetClusterConf(app.ClusterId) if err != nil { impl.logger.Errorw("error in fetching cluster detail", "err", err) return nil, err } - req := &AppDetailRequest{ + req := &gRPC.AppDetailRequest{ ClusterConfig: config, Namespace: app.Namespace, ReleaseName: app.ReleaseName, } history, err := impl.helmAppClient.GetDeploymentHistory(ctx, req) + if util.GetGRPCErrorDetailedMessage(err) == bean.ErrReleaseNotFound { + err = &util.ApiError{ + Code: constants.HelmReleaseNotFound, + InternalMessage: bean.ErrReleaseNotFound, + UserMessage: fmt.Sprintf("no release found with release name '%s'", req.ReleaseName), + } + } return history, err } -func (impl *HelmAppServiceImpl) GetValuesYaml(ctx context.Context, app *AppIdentifier) (*ReleaseInfo, error) { +func (impl *HelmAppServiceImpl) GetValuesYaml(ctx context.Context, app *AppIdentifier) (*gRPC.ReleaseInfo, error) { config, err := impl.GetClusterConf(app.ClusterId) if err != nil { impl.logger.Errorw("error in fetching cluster detail", "err", err) return nil, err } - req := &AppDetailRequest{ + req := &gRPC.AppDetailRequest{ ClusterConfig: config, Namespace: app.Namespace, ReleaseName: app.ReleaseName, @@ -390,11 +400,11 @@ func (impl *HelmAppServiceImpl) GetDesiredManifest(ctx context.Context, app *App return nil, err } - req := &ObjectRequest{ + req := &gRPC.ObjectRequest{ ClusterConfig: config, ReleaseName: app.ReleaseName, ReleaseNamespace: app.Namespace, - ObjectIdentifier: &ObjectIdentifier{ + ObjectIdentifier: &gRPC.ObjectIdentifier{ Group: resource.GetGroup(), Kind: resource.GetKind(), Version: resource.GetVersion(), @@ -513,7 +523,7 @@ func (impl *HelmAppServiceImpl) DeleteApplication(ctx context.Context, app *AppI return nil, models.NamespaceNotExistError{Err: fmt.Errorf("namespace %s does not exist", app.Namespace)} } - req := &ReleaseIdentifier{ + req := &gRPC.ReleaseIdentifier{ ClusterConfig: config, ReleaseName: app.ReleaseName, ReleaseNamespace: app.Namespace, @@ -564,15 +574,15 @@ func (impl *HelmAppServiceImpl) checkIfNsExists(app *AppIdentifier) (bool, error return exists, nil } -func (impl *HelmAppServiceImpl) UpdateApplication(ctx context.Context, app *AppIdentifier, request *UpdateApplicationRequestDto) (*openapi.UpdateReleaseResponse, error) { +func (impl *HelmAppServiceImpl) UpdateApplication(ctx context.Context, app *AppIdentifier, request *bean.UpdateApplicationRequestDto) (*openapi.UpdateReleaseResponse, error) { config, err := impl.GetClusterConf(app.ClusterId) if err != nil { impl.logger.Errorw("error in fetching cluster detail", "clusterId", app.ClusterId, "err", err) return nil, err } - req := &UpgradeReleaseRequest{ - ReleaseIdentifier: &ReleaseIdentifier{ + req := &gRPC.UpgradeReleaseRequest{ + ReleaseIdentifier: &gRPC.ReleaseIdentifier{ ClusterConfig: config, ReleaseName: app.ReleaseName, ReleaseNamespace: app.Namespace, @@ -600,8 +610,8 @@ func (impl *HelmAppServiceImpl) GetDeploymentDetail(ctx context.Context, app *Ap return nil, err } - req := &DeploymentDetailRequest{ - ReleaseIdentifier: &ReleaseIdentifier{ + req := &gRPC.DeploymentDetailRequest{ + ReleaseIdentifier: &gRPC.ReleaseIdentifier{ ClusterConfig: config, ReleaseName: app.ReleaseName, ReleaseNamespace: app.Namespace, @@ -623,7 +633,7 @@ func (impl *HelmAppServiceImpl) GetDeploymentDetail(ctx context.Context, app *Ap return response, nil } -func (impl *HelmAppServiceImpl) InstallRelease(ctx context.Context, clusterId int, installReleaseRequest *InstallReleaseRequest) (*InstallReleaseResponse, error) { +func (impl *HelmAppServiceImpl) InstallRelease(ctx context.Context, clusterId int, installReleaseRequest *gRPC.InstallReleaseRequest) (*gRPC.InstallReleaseResponse, error) { config, err := impl.GetClusterConf(clusterId) if err != nil { impl.logger.Errorw("error in fetching cluster detail", "clusterId", clusterId, "err", err) @@ -642,7 +652,7 @@ func (impl *HelmAppServiceImpl) InstallRelease(ctx context.Context, clusterId in } func (impl *HelmAppServiceImpl) UpdateApplicationWithChartInfo(ctx context.Context, clusterId int, - request *UpdateApplicationWithChartInfoRequestDto) (*openapi.UpdateReleaseResponse, error) { + request *bean.UpdateApplicationWithChartInfoRequestDto) (*openapi.UpdateReleaseResponse, error) { config, err := impl.GetClusterConf(clusterId) if err != nil { impl.logger.Errorw("error in fetching cluster detail", "clusterId", clusterId, "err", err) @@ -671,7 +681,7 @@ func (impl *HelmAppServiceImpl) IsReleaseInstalled(ctx context.Context, app *App return false, err } - req := &ReleaseIdentifier{ + req := &gRPC.ReleaseIdentifier{ ClusterConfig: config, ReleaseName: app.ReleaseName, ReleaseNamespace: app.Namespace, @@ -693,8 +703,8 @@ func (impl *HelmAppServiceImpl) RollbackRelease(ctx context.Context, app *AppIde return false, err } - req := &RollbackReleaseRequest{ - ReleaseIdentifier: &ReleaseIdentifier{ + req := &gRPC.RollbackReleaseRequest{ + ReleaseIdentifier: &gRPC.ReleaseIdentifier{ ClusterConfig: config, ReleaseName: app.ReleaseName, ReleaseNamespace: app.Namespace, @@ -720,7 +730,7 @@ func (impl *HelmAppServiceImpl) GetDevtronHelmAppIdentifier() *AppIdentifier { } func (impl *HelmAppServiceImpl) UpdateApplicationWithChartInfoWithExtraValues(ctx context.Context, appIdentifier *AppIdentifier, - chartRepository *ChartRepository, extraValues map[string]interface{}, extraValuesYamlUrl string, useLatestChartVersion bool) (*openapi.UpdateReleaseResponse, error) { + chartRepository *gRPC.ChartRepository, extraValues map[string]interface{}, extraValuesYamlUrl string, useLatestChartVersion bool) (*openapi.UpdateReleaseResponse, error) { // get release info releaseInfo, err := impl.GetValuesYaml(context.Background(), appIdentifier) @@ -794,9 +804,9 @@ func (impl *HelmAppServiceImpl) UpdateApplicationWithChartInfoWithExtraValues(ct // update in helm - updateReleaseRequest := &UpdateApplicationWithChartInfoRequestDto{ - InstallReleaseRequest: &InstallReleaseRequest{ - ReleaseIdentifier: &ReleaseIdentifier{ + updateReleaseRequest := &bean.UpdateApplicationWithChartInfoRequestDto{ + InstallReleaseRequest: &gRPC.InstallReleaseRequest{ + ReleaseIdentifier: &gRPC.ReleaseIdentifier{ ReleaseName: appIdentifier.ReleaseName, ReleaseNamespace: appIdentifier.Namespace, }, @@ -804,7 +814,7 @@ func (impl *HelmAppServiceImpl) UpdateApplicationWithChartInfoWithExtraValues(ct ValuesYaml: string(mergedValuesYamlByteArr), ChartRepository: chartRepository, }, - SourceAppType: SOURCE_UNKNOWN, + SourceAppType: bean.SOURCE_UNKNOWN, } if !useLatestChartVersion { updateReleaseRequest.ChartVersion = releaseInfo.DeployedAppDetail.ChartVersion @@ -865,8 +875,8 @@ func (impl *HelmAppServiceImpl) TemplateChart(ctx context.Context, templateChart return nil, err } var IsOCIRepo bool - var registryCredential *RegistryCredential - var chartRepository *ChartRepository + var registryCredential *gRPC.RegistryCredential + var chartRepository *gRPC.ChartRepository dockerRegistryId := appStoreAppVersion.AppStore.DockerArtifactStoreId if dockerRegistryId != "" { ociRegistryConfigs := appStoreAppVersion.AppStore.DockerArtifactStore.OCIRegistryConfig @@ -882,7 +892,7 @@ func (impl *HelmAppServiceImpl) TemplateChart(ctx context.Context, templateChart } } IsOCIRepo = true - registryCredential = &RegistryCredential{ + registryCredential = &gRPC.RegistryCredential{ RegistryUrl: appStoreAppVersion.AppStore.DockerArtifactStore.RegistryURL, Username: appStoreAppVersion.AppStore.DockerArtifactStore.Username, Password: appStoreAppVersion.AppStore.DockerArtifactStore.Password, @@ -894,7 +904,7 @@ func (impl *HelmAppServiceImpl) TemplateChart(ctx context.Context, templateChart IsPublic: ociRegistryConfig.IsPublic, } } else { - chartRepository = &ChartRepository{ + chartRepository = &gRPC.ChartRepository{ Name: appStoreAppVersion.AppStore.ChartRepo.Name, Url: appStoreAppVersion.AppStore.ChartRepo.Url, Username: appStoreAppVersion.AppStore.ChartRepo.UserName, @@ -902,13 +912,13 @@ func (impl *HelmAppServiceImpl) TemplateChart(ctx context.Context, templateChart } } - installReleaseRequest := &InstallReleaseRequest{ + installReleaseRequest := &gRPC.InstallReleaseRequest{ ChartName: appStoreAppVersion.Name, ChartVersion: appStoreAppVersion.Version, ValuesYaml: *templateChartRequest.ValuesYaml, K8SVersion: k8sServerVersion.String(), ChartRepository: chartRepository, - ReleaseIdentifier: &ReleaseIdentifier{ + ReleaseIdentifier: &gRPC.ReleaseIdentifier{ ReleaseNamespace: *templateChartRequest.Namespace, ReleaseName: *templateChartRequest.ReleaseName, }, @@ -936,7 +946,7 @@ func (impl *HelmAppServiceImpl) TemplateChart(ctx context.Context, templateChart return response, nil } -func (impl *HelmAppServiceImpl) GetNotes(ctx context.Context, request *InstallReleaseRequest) (string, error) { +func (impl *HelmAppServiceImpl) GetNotes(ctx context.Context, request *gRPC.InstallReleaseRequest) (string, error) { var notesTxt string response, err := impl.helmAppClient.GetNotes(ctx, request) if err != nil { @@ -947,7 +957,7 @@ func (impl *HelmAppServiceImpl) GetNotes(ctx context.Context, request *InstallRe return notesTxt, err } -func (impl *HelmAppServiceImpl) ValidateOCIRegistry(ctx context.Context, OCIRegistryRequest *RegistryCredential) bool { +func (impl *HelmAppServiceImpl) ValidateOCIRegistry(ctx context.Context, OCIRegistryRequest *gRPC.RegistryCredential) bool { response, err := impl.helmAppClient.ValidateOCIRegistry(ctx, OCIRegistryRequest) if err != nil { impl.logger.Errorw("error in fetching chart", "err", err) @@ -985,7 +995,7 @@ func (impl *HelmAppServiceImpl) EncodeAppId(appIdentifier *AppIdentifier) string return fmt.Sprintf("%d|%s|%s", appIdentifier.ClusterId, appIdentifier.Namespace, appIdentifier.ReleaseName) } -func (impl *HelmAppServiceImpl) appListRespProtoTransformer(deployedApps *DeployedAppList, token string, helmAuth func(token string, object string) bool, helmCdPipelines []*pipelineConfig.Pipeline, installedHelmApps []*repository.InstalledApps) openapi.AppList { +func (impl *HelmAppServiceImpl) appListRespProtoTransformer(deployedApps *gRPC.DeployedAppList, token string, helmAuth func(token string, object string) bool, helmCdPipelines []*pipelineConfig.Pipeline, installedHelmApps []*repository.InstalledApps) openapi.AppList { applicationType := "HELM-APP" appList := openapi.AppList{ClusterIds: &[]int32{deployedApps.ClusterId}, ApplicationType: &applicationType} if deployedApps.Errored { @@ -1054,13 +1064,13 @@ func (impl *HelmAppServiceImpl) appListRespProtoTransformer(deployedApps *Deploy return appList } -func (impl *HelmAppServiceImpl) GetRevisionHistoryMaxValue(appType SourceAppType) int32 { +func (impl *HelmAppServiceImpl) GetRevisionHistoryMaxValue(appType bean.SourceAppType) int32 { switch appType { - case SOURCE_DEVTRON_APP: + case bean.SOURCE_DEVTRON_APP: return int32(impl.helmReleaseConfig.RevisionHistoryLimitDevtronApp) - case SOURCE_HELM_APP: + case bean.SOURCE_HELM_APP: return int32(impl.helmReleaseConfig.RevisionHistoryLimitHelmApp) - case SOURCE_EXTERNAL_HELM_APP: + case bean.SOURCE_EXTERNAL_HELM_APP: return int32(impl.helmReleaseConfig.RevisionHistoryLimitExternalHelmApp) default: return 0 diff --git a/api/helm-app/wire_helmApp.go b/api/helm-app/wire_helmApp.go index 82c492f049..f7a453ee3b 100644 --- a/api/helm-app/wire_helmApp.go +++ b/api/helm-app/wire_helmApp.go @@ -1,21 +1,23 @@ package client import ( + "github.com/devtron-labs/devtron/api/helm-app/gRPC" + "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/util/rbac" "github.com/google/wire" ) var HelmAppWireSet = wire.NewSet( - NewHelmAppClientImpl, - wire.Bind(new(HelmAppClient), new(*HelmAppClientImpl)), - GetHelmReleaseConfig, - NewHelmAppServiceImpl, - wire.Bind(new(HelmAppService), new(*HelmAppServiceImpl)), + gRPC.NewHelmAppClientImpl, + wire.Bind(new(gRPC.HelmAppClient), new(*gRPC.HelmAppClientImpl)), + service.GetHelmReleaseConfig, + service.NewHelmAppServiceImpl, + wire.Bind(new(service.HelmAppService), new(*service.HelmAppServiceImpl)), NewHelmAppRestHandlerImpl, wire.Bind(new(HelmAppRestHandler), new(*HelmAppRestHandlerImpl)), NewHelmAppRouterImpl, wire.Bind(new(HelmAppRouter), new(*HelmAppRouterImpl)), - GetConfig, + gRPC.GetConfig, rbac.NewEnforcerUtilHelmImpl, wire.Bind(new(rbac.EnforcerUtilHelm), new(*rbac.EnforcerUtilHelmImpl)), ) diff --git a/api/k8s/application/k8sApplicationRestHandler.go b/api/k8s/application/k8sApplicationRestHandler.go index 0fa90296ab..b0af48bea4 100644 --- a/api/k8s/application/k8sApplicationRestHandler.go +++ b/api/k8s/application/k8sApplicationRestHandler.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + client "github.com/devtron-labs/devtron/api/helm-app/service" "net/http" "strconv" "strings" @@ -14,7 +15,6 @@ import ( "github.com/devtron-labs/common-lib/utils/k8sObjectsUtil" "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/api/connector" - client "github.com/devtron-labs/devtron/api/helm-app" "github.com/devtron-labs/devtron/api/restHandler/common" util2 "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" diff --git a/api/restHandler/AppListingRestHandler.go b/api/restHandler/AppListingRestHandler.go index 870f54bae6..6c20e37b98 100644 --- a/api/restHandler/AppListingRestHandler.go +++ b/api/restHandler/AppListingRestHandler.go @@ -21,6 +21,8 @@ import ( "context" "encoding/json" "fmt" + "github.com/devtron-labs/devtron/api/helm-app/gRPC" + client "github.com/devtron-labs/devtron/api/helm-app/service" "net/http" "strconv" "strings" @@ -33,7 +35,6 @@ import ( "github.com/devtron-labs/common-lib/utils/k8s/health" k8sObjectUtils "github.com/devtron-labs/common-lib/utils/k8sObjectsUtil" "github.com/devtron-labs/devtron/api/bean" - client "github.com/devtron-labs/devtron/api/helm-app" "github.com/devtron-labs/devtron/api/restHandler/common" "github.com/devtron-labs/devtron/client/argocdServer/application" "github.com/devtron-labs/devtron/client/cron" @@ -90,16 +91,17 @@ type AppListingRestHandler interface { } type AppListingRestHandlerImpl struct { - application application.ServiceClient - appListingService app.AppListingService - teamService team.TeamService - enforcer casbin.Enforcer - pipeline pipeline.PipelineBuilder - logger *zap.SugaredLogger - enforcerUtil rbac.EnforcerUtil - deploymentGroupService deploymentGroup.DeploymentGroupService - userService user.UserService - helmAppClient client.HelmAppClient + application application.ServiceClient + appListingService app.AppListingService + teamService team.TeamService + enforcer casbin.Enforcer + pipeline pipeline.PipelineBuilder + logger *zap.SugaredLogger + enforcerUtil rbac.EnforcerUtil + deploymentGroupService deploymentGroup.DeploymentGroupService + userService user.UserService + // TODO fix me next + helmAppClient gRPC.HelmAppClient // TODO refactoring: use HelmAppService clusterService cluster.ClusterService helmAppService client.HelmAppService argoUserService argo.ArgoUserService @@ -137,7 +139,7 @@ func NewAppListingRestHandlerImpl(application application.ServiceClient, pipeline pipeline.PipelineBuilder, logger *zap.SugaredLogger, enforcerUtil rbac.EnforcerUtil, deploymentGroupService deploymentGroup.DeploymentGroupService, userService user.UserService, - helmAppClient client.HelmAppClient, clusterService cluster.ClusterService, helmAppService client.HelmAppService, + helmAppClient gRPC.HelmAppClient, clusterService cluster.ClusterService, helmAppService client.HelmAppService, argoUserService argo.ArgoUserService, k8sCommonService k8s.K8sCommonService, installedAppService service1.InstalledAppService, cdApplicationStatusUpdateHandler cron.CdApplicationStatusUpdateHandler, pipelineRepository pipelineConfig.PipelineRepository, @@ -1578,7 +1580,7 @@ func (handler AppListingRestHandlerImpl) fetchResourceTree(w http.ResponseWriter if err != nil { handler.logger.Errorw("error in fetching cluster detail", "err", err) } - req := &client.AppDetailRequest{ + req := &gRPC.AppDetailRequest{ ClusterConfig: config, Namespace: cdPipeline.Environment.Namespace, ReleaseName: cdPipeline.DeploymentAppName, diff --git a/api/restHandler/AppRestHandler.go b/api/restHandler/AppRestHandler.go index ed0b6ab4e0..c787fd4f66 100644 --- a/api/restHandler/AppRestHandler.go +++ b/api/restHandler/AppRestHandler.go @@ -19,11 +19,11 @@ package restHandler import ( "encoding/json" + client "github.com/devtron-labs/devtron/api/helm-app/service" "net/http" "strconv" "strings" - client "github.com/devtron-labs/devtron/api/helm-app" "github.com/devtron-labs/devtron/api/restHandler/common" "github.com/devtron-labs/devtron/internal/sql/repository/helper" "github.com/devtron-labs/devtron/pkg/app" diff --git a/client/telemetry/TelemetryEventClient.go b/client/telemetry/TelemetryEventClient.go index 7bb093ac6c..3c36e51f77 100644 --- a/client/telemetry/TelemetryEventClient.go +++ b/client/telemetry/TelemetryEventClient.go @@ -5,12 +5,12 @@ import ( "encoding/base64" "encoding/json" "fmt" + "github.com/devtron-labs/devtron/api/helm-app/gRPC" "net/http" "time" "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/devtron/api/bean" - client "github.com/devtron-labs/devtron/api/helm-app" "github.com/devtron-labs/devtron/internal/sql/repository" repository2 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" "github.com/devtron-labs/devtron/pkg/attributes" @@ -53,7 +53,7 @@ type TelemetryEventClientImpl struct { moduleRepository moduleRepo.ModuleRepository serverDataStore *serverDataStore.ServerDataStore userAuditService user2.UserAuditService - helmAppClient client.HelmAppClient + helmAppClient gRPC.HelmAppClient InstalledAppRepository repository2.InstalledAppRepository userAttributesRepository repository.UserAttributesRepository } @@ -70,7 +70,7 @@ type TelemetryEventClient interface { func NewTelemetryEventClientImpl(logger *zap.SugaredLogger, client *http.Client, clusterService cluster.ClusterService, K8sUtil *k8s.K8sUtil, aCDAuthConfig *util3.ACDAuthConfig, userService user2.UserService, attributeRepo repository.AttributesRepository, ssoLoginService sso.SSOLoginService, - PosthogClient *PosthogClient, moduleRepository moduleRepo.ModuleRepository, serverDataStore *serverDataStore.ServerDataStore, userAuditService user2.UserAuditService, helmAppClient client.HelmAppClient, InstalledAppRepository repository2.InstalledAppRepository) (*TelemetryEventClientImpl, error) { + PosthogClient *PosthogClient, moduleRepository moduleRepo.ModuleRepository, serverDataStore *serverDataStore.ServerDataStore, userAuditService user2.UserAuditService, helmAppClient gRPC.HelmAppClient, InstalledAppRepository repository2.InstalledAppRepository) (*TelemetryEventClientImpl, error) { cron := cron.New( cron.WithChain()) cron.Start() @@ -219,8 +219,8 @@ func (impl *TelemetryEventClientImpl) SummaryDetailsForTelemetry() (cluster []cl ExternalHelmAppClusterCount = make(map[int32]int) for _, clusterDetail := range clusters { - req := &client.AppListRequest{} - config := &client.ClusterConfig{ + req := &gRPC.AppListRequest{} + config := &gRPC.ClusterConfig{ ApiServerUrl: clusterDetail.ServerUrl, Token: clusterDetail.Config[k8s.BearerToken], ClusterId: int32(clusterDetail.Id), diff --git a/client/telemetry/TelemetryEventClientExtended.go b/client/telemetry/TelemetryEventClientExtended.go index c719e12513..e427edb0df 100644 --- a/client/telemetry/TelemetryEventClientExtended.go +++ b/client/telemetry/TelemetryEventClientExtended.go @@ -2,11 +2,11 @@ package telemetry import ( "encoding/json" + client "github.com/devtron-labs/devtron/api/helm-app/gRPC" "net/http" "time" util2 "github.com/devtron-labs/common-lib/utils/k8s" - client "github.com/devtron-labs/devtron/api/helm-app" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/app" dockerRegistryRepository "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index f304d374d9..819b9b7363 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -24,6 +24,8 @@ import ( "github.com/devtron-labs/devtron/api/dashboardEvent" externalLink2 "github.com/devtron-labs/devtron/api/externalLink" client2 "github.com/devtron-labs/devtron/api/helm-app" + "github.com/devtron-labs/devtron/api/helm-app/gRPC" + "github.com/devtron-labs/devtron/api/helm-app/service" application2 "github.com/devtron-labs/devtron/api/k8s/application" capacity2 "github.com/devtron-labs/devtron/api/k8s/capacity" module2 "github.com/devtron-labs/devtron/api/module" @@ -45,16 +47,16 @@ import ( "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/apiToken" app2 "github.com/devtron-labs/devtron/pkg/app" - repository8 "github.com/devtron-labs/devtron/pkg/appStore/chartGroup/repository" + repository7 "github.com/devtron-labs/devtron/pkg/appStore/chartGroup/repository" "github.com/devtron-labs/devtron/pkg/appStore/chartProvider" "github.com/devtron-labs/devtron/pkg/appStore/deployment/common" repository4 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" - service3 "github.com/devtron-labs/devtron/pkg/appStore/deployment/service" + service2 "github.com/devtron-labs/devtron/pkg/appStore/deployment/service" "github.com/devtron-labs/devtron/pkg/appStore/deployment/tool" "github.com/devtron-labs/devtron/pkg/appStore/discover/repository" - "github.com/devtron-labs/devtron/pkg/appStore/discover/service" + service3 "github.com/devtron-labs/devtron/pkg/appStore/discover/service" "github.com/devtron-labs/devtron/pkg/appStore/values/repository" - service2 "github.com/devtron-labs/devtron/pkg/appStore/values/service" + service4 "github.com/devtron-labs/devtron/pkg/appStore/values/service" "github.com/devtron-labs/devtron/pkg/attributes" "github.com/devtron-labs/devtron/pkg/auth/authentication" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" @@ -77,7 +79,7 @@ import ( "github.com/devtron-labs/devtron/pkg/k8s/capacity" "github.com/devtron-labs/devtron/pkg/k8s/informer" "github.com/devtron-labs/devtron/pkg/kubernetesResourceAuditLogs" - repository7 "github.com/devtron-labs/devtron/pkg/kubernetesResourceAuditLogs/repository" + repository8 "github.com/devtron-labs/devtron/pkg/kubernetesResourceAuditLogs/repository" "github.com/devtron-labs/devtron/pkg/module" "github.com/devtron-labs/devtron/pkg/module/repo" "github.com/devtron-labs/devtron/pkg/module/store" @@ -185,22 +187,22 @@ func InitializeApp() (*App, error) { } chartRepositoryServiceImpl := chartRepo.NewChartRepositoryServiceImpl(sugaredLogger, chartRepoRepositoryImpl, k8sUtil, clusterServiceImpl, acdAuthConfig, httpClient, serverEnvConfigServerEnvConfig) installedAppRepositoryImpl := repository4.NewInstalledAppRepositoryImpl(sugaredLogger, db) - helmClientConfig, err := client2.GetConfig() + helmClientConfig, err := gRPC.GetConfig() if err != nil { return nil, err } - helmAppClientImpl := client2.NewHelmAppClientImpl(sugaredLogger, helmClientConfig) + helmAppClientImpl := gRPC.NewHelmAppClientImpl(sugaredLogger, helmClientConfig) pumpImpl := connector.NewPumpImpl(sugaredLogger) appRepositoryImpl := app.NewAppRepositoryImpl(db, sugaredLogger) enforcerUtilHelmImpl := rbac.NewEnforcerUtilHelmImpl(sugaredLogger, clusterRepositoryImpl, teamRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, installedAppRepositoryImpl) serverDataStoreServerDataStore := serverDataStore.InitServerDataStore() appStoreApplicationVersionRepositoryImpl := appStoreDiscoverRepository.NewAppStoreApplicationVersionRepositoryImpl(sugaredLogger, db) pipelineRepositoryImpl := pipelineConfig.NewPipelineRepositoryImpl(db, sugaredLogger) - helmReleaseConfig, err := client2.GetHelmReleaseConfig() + helmReleaseConfig, err := service.GetHelmReleaseConfig() if err != nil { return nil, err } - helmAppServiceImpl := client2.NewHelmAppServiceImpl(sugaredLogger, clusterServiceImpl, helmAppClientImpl, pumpImpl, enforcerUtilHelmImpl, serverDataStoreServerDataStore, serverEnvConfigServerEnvConfig, appStoreApplicationVersionRepositoryImpl, environmentServiceImpl, pipelineRepositoryImpl, installedAppRepositoryImpl, appRepositoryImpl, clusterRepositoryImpl, k8sUtil, helmReleaseConfig) + helmAppServiceImpl := service.NewHelmAppServiceImpl(sugaredLogger, clusterServiceImpl, helmAppClientImpl, pumpImpl, enforcerUtilHelmImpl, serverDataStoreServerDataStore, serverEnvConfigServerEnvConfig, appStoreApplicationVersionRepositoryImpl, environmentServiceImpl, pipelineRepositoryImpl, installedAppRepositoryImpl, appRepositoryImpl, clusterRepositoryImpl, k8sUtil, helmReleaseConfig) dockerArtifactStoreRepositoryImpl := repository5.NewDockerArtifactStoreRepositoryImpl(db) dockerRegistryIpsConfigRepositoryImpl := repository5.NewDockerRegistryIpsConfigRepositoryImpl(db) ociRegistryConfigRepositoryImpl := repository5.NewOCIRegistryConfigRepositoryImpl(db) @@ -231,10 +233,18 @@ func InitializeApp() (*App, error) { return nil, err } dashboardRouterImpl := dashboard.NewDashboardRouterImpl(sugaredLogger, dashboardConfig) + chartGroupDeploymentRepositoryImpl := repository7.NewChartGroupDeploymentRepositoryImpl(db, sugaredLogger) + clusterInstalledAppsRepositoryImpl := repository4.NewClusterInstalledAppsRepositoryImpl(db, sugaredLogger) + appStoreDeploymentHelmServiceImpl := appStoreDeploymentTool.NewAppStoreDeploymentHelmServiceImpl(sugaredLogger, helmAppServiceImpl, appStoreApplicationVersionRepositoryImpl, helmAppClientImpl, installedAppRepositoryImpl, ociRegistryConfigRepositoryImpl) chartTemplateServiceImpl := util.NewChartTemplateServiceImpl(sugaredLogger) + appStoreDeploymentCommonServiceImpl := appStoreDeploymentCommon.NewAppStoreDeploymentCommonServiceImpl(sugaredLogger, appStoreApplicationVersionRepositoryImpl, chartTemplateServiceImpl) + installedAppVersionHistoryRepositoryImpl := repository4.NewInstalledAppVersionHistoryRepositoryImpl(sugaredLogger, db) gitOpsConfigRepositoryImpl := repository3.NewGitOpsConfigRepositoryImpl(sugaredLogger, db) - gitCliUtil := util.NewGitCliUtil(sugaredLogger) - gitFactory, err := util.NewGitFactory(sugaredLogger, gitOpsConfigRepositoryImpl, gitCliUtil) + deploymentServiceTypeConfig, err := service2.GetDeploymentServiceTypeConfig() + if err != nil { + return nil, err + } + acdConfig, err := argocdServer.GetACDDeploymentConfig() if err != nil { return nil, err } @@ -243,15 +253,20 @@ func InitializeApp() (*App, error) { return nil, err } gitOpsConfigReadServiceImpl := config.NewGitOpsConfigReadServiceImpl(sugaredLogger, gitOpsConfigRepositoryImpl, userServiceImpl, globalEnvVariables) + gitCliUtil := util.NewGitCliUtil(sugaredLogger) + gitFactory, err := util.NewGitFactory(sugaredLogger, gitOpsConfigRepositoryImpl, gitCliUtil) + if err != nil { + return nil, err + } gitOpsRemoteOperationServiceImpl := remote.NewGitOpsRemoteOperationServiceImpl(sugaredLogger, gitFactory, gitOpsConfigReadServiceImpl, chartTemplateServiceImpl) - appStoreDeploymentCommonServiceImpl := appStoreDeploymentCommon.NewAppStoreDeploymentCommonServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, chartTemplateServiceImpl, gitFactory, gitOpsConfigReadServiceImpl, gitOpsRemoteOperationServiceImpl) + appStoreDeploymentServiceImpl := service2.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, clusterInstalledAppsRepositoryImpl, appRepositoryImpl, appStoreDeploymentHelmServiceImpl, appStoreDeploymentHelmServiceImpl, environmentServiceImpl, clusterServiceImpl, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, installedAppVersionHistoryRepositoryImpl, gitOpsConfigRepositoryImpl, deploymentServiceTypeConfig, acdConfig, gitOpsConfigReadServiceImpl, gitOpsRemoteOperationServiceImpl) attributesServiceImpl := attributes.NewAttributesServiceImpl(sugaredLogger, attributesRepositoryImpl) - helmAppRestHandlerImpl := client2.NewHelmAppRestHandlerImpl(sugaredLogger, helmAppServiceImpl, enforcerImpl, clusterServiceImpl, enforcerUtilHelmImpl, appStoreDeploymentCommonServiceImpl, userServiceImpl, attributesServiceImpl, serverEnvConfigServerEnvConfig) + helmAppRestHandlerImpl := client2.NewHelmAppRestHandlerImpl(sugaredLogger, helmAppServiceImpl, enforcerImpl, clusterServiceImpl, enforcerUtilHelmImpl, appStoreDeploymentServiceImpl, userServiceImpl, attributesServiceImpl, serverEnvConfigServerEnvConfig) helmAppRouterImpl := client2.NewHelmAppRouterImpl(helmAppRestHandlerImpl) k8sCommonServiceImpl := k8s2.NewK8sCommonServiceImpl(sugaredLogger, k8sUtil, clusterServiceImpl) environmentRestHandlerImpl := cluster2.NewEnvironmentRestHandlerImpl(environmentServiceImpl, sugaredLogger, userServiceImpl, validate, enforcerImpl, deleteServiceImpl, k8sUtil, k8sCommonServiceImpl) environmentRouterImpl := cluster2.NewEnvironmentRouterImpl(environmentRestHandlerImpl) - k8sResourceHistoryRepositoryImpl := repository7.NewK8sResourceHistoryRepositoryImpl(db, sugaredLogger) + k8sResourceHistoryRepositoryImpl := repository8.NewK8sResourceHistoryRepositoryImpl(db, sugaredLogger) k8sResourceHistoryServiceImpl := kubernetesResourceAuditLogs.Newk8sResourceHistoryServiceImpl(k8sResourceHistoryRepositoryImpl, sugaredLogger, appRepositoryImpl, environmentRepositoryImpl) ephemeralContainersRepositoryImpl := repository2.NewEphemeralContainersRepositoryImpl(db) ephemeralContainerServiceImpl := cluster.NewEphemeralContainerServiceImpl(ephemeralContainersRepositoryImpl, sugaredLogger) @@ -266,26 +281,13 @@ func InitializeApp() (*App, error) { k8sApplicationRouterImpl := application2.NewK8sApplicationRouterImpl(k8sApplicationRestHandlerImpl) chartRepositoryRestHandlerImpl := chartRepo2.NewChartRepositoryRestHandlerImpl(sugaredLogger, userServiceImpl, chartRepositoryServiceImpl, enforcerImpl, validate, deleteServiceImpl, attributesServiceImpl) chartRepositoryRouterImpl := chartRepo2.NewChartRepositoryRouterImpl(chartRepositoryRestHandlerImpl) - appStoreServiceImpl := service.NewAppStoreServiceImpl(sugaredLogger, appStoreApplicationVersionRepositoryImpl) + appStoreServiceImpl := service3.NewAppStoreServiceImpl(sugaredLogger, appStoreApplicationVersionRepositoryImpl) appStoreRestHandlerImpl := appStoreDiscover.NewAppStoreRestHandlerImpl(sugaredLogger, userServiceImpl, appStoreServiceImpl, enforcerImpl) appStoreDiscoverRouterImpl := appStoreDiscover.NewAppStoreDiscoverRouterImpl(appStoreRestHandlerImpl) appStoreVersionValuesRepositoryImpl := appStoreValuesRepository.NewAppStoreVersionValuesRepositoryImpl(sugaredLogger, db) - appStoreValuesServiceImpl := service2.NewAppStoreValuesServiceImpl(sugaredLogger, appStoreApplicationVersionRepositoryImpl, installedAppRepositoryImpl, appStoreVersionValuesRepositoryImpl, userServiceImpl) + appStoreValuesServiceImpl := service4.NewAppStoreValuesServiceImpl(sugaredLogger, appStoreApplicationVersionRepositoryImpl, installedAppRepositoryImpl, appStoreVersionValuesRepositoryImpl, userServiceImpl) appStoreValuesRestHandlerImpl := appStoreValues.NewAppStoreValuesRestHandlerImpl(sugaredLogger, userServiceImpl, appStoreValuesServiceImpl) appStoreValuesRouterImpl := appStoreValues.NewAppStoreValuesRouterImpl(appStoreValuesRestHandlerImpl) - chartGroupDeploymentRepositoryImpl := repository8.NewChartGroupDeploymentRepositoryImpl(db, sugaredLogger) - clusterInstalledAppsRepositoryImpl := repository4.NewClusterInstalledAppsRepositoryImpl(db, sugaredLogger) - appStoreDeploymentHelmServiceImpl := appStoreDeploymentTool.NewAppStoreDeploymentHelmServiceImpl(sugaredLogger, helmAppServiceImpl, appStoreApplicationVersionRepositoryImpl, helmAppClientImpl, installedAppRepositoryImpl, appStoreDeploymentCommonServiceImpl, ociRegistryConfigRepositoryImpl, gitOpsRemoteOperationServiceImpl) - installedAppVersionHistoryRepositoryImpl := repository4.NewInstalledAppVersionHistoryRepositoryImpl(sugaredLogger, db) - deploymentServiceTypeConfig, err := service3.GetDeploymentServiceTypeConfig() - if err != nil { - return nil, err - } - acdConfig, err := argocdServer.GetACDDeploymentConfig() - if err != nil { - return nil, err - } - appStoreDeploymentServiceImpl := service3.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, clusterInstalledAppsRepositoryImpl, appRepositoryImpl, appStoreDeploymentHelmServiceImpl, appStoreDeploymentHelmServiceImpl, environmentServiceImpl, clusterServiceImpl, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, installedAppVersionHistoryRepositoryImpl, gitOpsConfigRepositoryImpl, deploymentServiceTypeConfig, acdConfig, gitOpsConfigReadServiceImpl, gitOpsRemoteOperationServiceImpl) appStoreDeploymentRestHandlerImpl := appStoreDeployment.NewAppStoreDeploymentRestHandlerImpl(sugaredLogger, userServiceImpl, enforcerImpl, enforcerUtilImpl, enforcerUtilHelmImpl, appStoreDeploymentServiceImpl, validate, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, helmUserServiceImpl, attributesServiceImpl) appStoreDeploymentRouterImpl := appStoreDeployment.NewAppStoreDeploymentRouterImpl(appStoreDeploymentRestHandlerImpl) chartProviderServiceImpl := chartProvider.NewChartProviderServiceImpl(sugaredLogger, chartRepoRepositoryImpl, chartRepositoryServiceImpl, dockerArtifactStoreRepositoryImpl, ociRegistryConfigRepositoryImpl) @@ -304,7 +306,7 @@ func InitializeApp() (*App, error) { } dashboardTelemetryRestHandlerImpl := dashboardEvent.NewDashboardTelemetryRestHandlerImpl(sugaredLogger, telemetryEventClientImpl) dashboardTelemetryRouterImpl := dashboardEvent.NewDashboardTelemetryRouterImpl(dashboardTelemetryRestHandlerImpl) - commonDeploymentRestHandlerImpl := appStoreDeployment.NewCommonDeploymentRestHandlerImpl(sugaredLogger, userServiceImpl, enforcerImpl, enforcerUtilImpl, enforcerUtilHelmImpl, appStoreDeploymentServiceImpl, validate, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, helmAppRestHandlerImpl, helmUserServiceImpl) + commonDeploymentRestHandlerImpl := appStoreDeployment.NewCommonDeploymentRestHandlerImpl(sugaredLogger, userServiceImpl, enforcerImpl, enforcerUtilImpl, enforcerUtilHelmImpl, appStoreDeploymentServiceImpl, validate, helmAppServiceImpl, helmAppRestHandlerImpl, helmUserServiceImpl) commonDeploymentRouterImpl := appStoreDeployment.NewCommonDeploymentRouterImpl(commonDeploymentRestHandlerImpl) externalLinkMonitoringToolRepositoryImpl := externalLink.NewExternalLinkMonitoringToolRepositoryImpl(db) externalLinkIdentifierMappingRepositoryImpl := externalLink.NewExternalLinkIdentifierMappingRepositoryImpl(db) diff --git a/internal/util/ChartService.go b/internal/util/ChartService.go index f904ed4490..84aa11e0dc 100644 --- a/internal/util/ChartService.go +++ b/internal/util/ChartService.go @@ -21,6 +21,7 @@ import ( "compress/gzip" "context" "fmt" + util2 "github.com/devtron-labs/devtron/pkg/appStore/util" "io/ioutil" "math/rand" "os" @@ -41,7 +42,7 @@ import ( const ( PIPELINE_DEPLOYMENT_TYPE_ACD = "argo_cd" PIPELINE_DEPLOYMENT_TYPE_HELM = "helm" - PIPELINE_DEPLOYMENT_TYPE_MANIFEST_DOWNLOAD = "manifest_download" + PIPELINE_DEPLOYMENT_TYPE_MANIFEST_DOWNLOAD = "manifest_download" // TODO refactoring: This const belongs to enterprise only ChartWorkingDirPath = "/tmp/charts/" ) @@ -67,6 +68,8 @@ type ChartTemplateService interface { LoadChartFromDir(dir string) (*chart.Chart, error) CreateZipFileForChart(chart *chart.Chart, outputChartPathDir string) ([]byte, error) PackageChart(tempReferenceTemplateDir string, chartMetaData *chart.Metadata) (*string, string, error) + // AddConfigFileToChart will override requirements.yaml file in chart + AddConfigFileToChart(config *ChartConfig, dir string, clonedDir string) error } type ChartTemplateServiceImpl struct { randSource rand.Source @@ -417,6 +420,28 @@ func (impl ChartTemplateServiceImpl) CreateZipFileForChart(chart *chart.Chart, o return chartBytesArr, nil } +func (impl ChartTemplateServiceImpl) AddConfigFileToChart(config *ChartConfig, dir string, clonedDir string) error { + filePath := filepath.Join(clonedDir, config.FileName) + file, err := os.Create(filePath) + if err != nil { + impl.logger.Errorw("error in creating file", "err", err, "fileName", config.FileName) + return err + } + defer file.Close() + _, err = file.Write([]byte(config.FileContent)) + if err != nil { + impl.logger.Errorw("error in writing file content", "err", err, "fileName", config.FileName) + return err + } + destinationFilePath := filepath.Join(dir, config.FileName) + err = util2.MoveFileToDestination(filePath, destinationFilePath) + if err != nil { + impl.logger.Errorw("error in moving file from source to destination", "err", err) + return err + } + return nil +} + func IsHelmApp(deploymentAppType string) bool { return deploymentAppType == PIPELINE_DEPLOYMENT_TYPE_HELM } @@ -425,6 +450,7 @@ func IsAcdApp(deploymentAppType string) bool { return deploymentAppType == PIPELINE_DEPLOYMENT_TYPE_ACD } +// TODO refactoring: This feature belongs to enterprise only func IsManifestDownload(deploymentAppType string) bool { return deploymentAppType == PIPELINE_DEPLOYMENT_TYPE_MANIFEST_DOWNLOAD } diff --git a/pkg/app/integrationTest/AppService_test.go b/pkg/app/integrationTest/AppService_test.go index 03ed6b6423..de06a34394 100644 --- a/pkg/app/integrationTest/AppService_test.go +++ b/pkg/app/integrationTest/AppService_test.go @@ -4,6 +4,8 @@ import ( "encoding/csv" "encoding/json" "fmt" + "github.com/devtron-labs/devtron/api/helm-app/gRPC" + client "github.com/devtron-labs/devtron/api/helm-app/service" "log" "os" "strconv" @@ -12,7 +14,6 @@ import ( "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" pubsub "github.com/devtron-labs/common-lib/pubsub-lib" - client "github.com/devtron-labs/devtron/api/helm-app" client1 "github.com/devtron-labs/devtron/client/events" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/app" @@ -130,11 +131,11 @@ func InitAppService() *app2.AppServiceImpl { moduleActionAuditLogRepository := module.NewModuleActionAuditLogRepositoryImpl(dbConnection) clusterRepository := repository1.NewClusterRepositoryImpl(dbConnection, logger) clusterService := cluster.NewClusterServiceImplExtended(clusterRepository, nil, nil, logger, nil, nil, nil, nil, nil, nil, nil, nil) - helmClientConfig, err := client.GetConfig() + helmClientConfig, err := gRPC.GetConfig() if err != nil { log.Fatal("error in getting server helm client config, AppService_test", "err", err) } - helmAppClient := client.NewHelmAppClientImpl(logger, helmClientConfig) + helmAppClient := gRPC.NewHelmAppClientImpl(logger, helmClientConfig) helmAppService := client.NewHelmAppServiceImpl(logger, clusterService, helmAppClient, nil, nil, nil, serverEnvConfig, nil, nil, nil, nil, nil, nil, nil, nil) moduleService := module.NewModuleServiceImpl(logger, serverEnvConfig, moduleRepositoryImpl, moduleActionAuditLogRepository, helmAppService, nil, nil, nil, nil, nil, nil, nil) eventClient := client1.NewEventRESTClientImpl(logger, httpClient, eventClientConfig, pubSubClient, ciPipelineRepositoryImpl, diff --git a/pkg/appStore/bean/bean.go b/pkg/appStore/bean/bean.go index c8954dffb8..86456bcfb9 100644 --- a/pkg/appStore/bean/bean.go +++ b/pkg/appStore/bean/bean.go @@ -351,15 +351,6 @@ func (a AppstoreDeploymentStatus) String() string { "HELM_SUCCESS"}[a] } -type PushChartToGitRequestDTO struct { - AppName string - EnvName string - ChartAppStoreName string - RepoURL string - TempChartRefDir string - UserId int32 -} - type HelmReleaseStatusConfig struct { InstallAppVersionHistoryId int Message string diff --git a/pkg/appStore/deployment/adapter/Adapter.go b/pkg/appStore/deployment/adapter/Adapter.go new file mode 100644 index 0000000000..32cfe7f161 --- /dev/null +++ b/pkg/appStore/deployment/adapter/Adapter.go @@ -0,0 +1,26 @@ +package adapter + +import ( + "github.com/devtron-labs/devtron/internal/util" + appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote/bean" + "k8s.io/helm/pkg/proto/hapi/chart" +) + +func ParseChartGitPushRequest(installAppRequestDTO *appStoreBean.InstallAppVersionDTO, repoURl string, tempRefChart string) *bean.PushChartToGitRequestDTO { + return &bean.PushChartToGitRequestDTO{ + AppName: installAppRequestDTO.AppName, + EnvName: installAppRequestDTO.Environment.Name, + ChartAppStoreName: installAppRequestDTO.AppStoreName, + RepoURL: repoURl, + TempChartRefDir: tempRefChart, + UserId: installAppRequestDTO.UserId, + } +} + +func ParseChartCreateRequest(installAppRequestDTO *appStoreBean.InstallAppVersionDTO, chartPath string) *util.ChartCreateRequest { + return &util.ChartCreateRequest{ChartMetaData: &chart.Metadata{ + Name: installAppRequestDTO.AppName, + Version: "1.0.1", + }, ChartPath: chartPath} +} diff --git a/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go b/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go index 59554f7d86..5000cb125e 100644 --- a/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go +++ b/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go @@ -19,147 +19,40 @@ package appStoreDeploymentCommon import ( "encoding/json" - "fmt" - "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/util" - "github.com/devtron-labs/devtron/pkg/appStore/adapter" appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" - "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" appStoreDiscoverRepository "github.com/devtron-labs/devtron/pkg/appStore/discover/repository" - util2 "github.com/devtron-labs/devtron/pkg/appStore/util" - repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" - commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" - "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" - "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote" - "github.com/go-pg/pg" "go.uber.org/zap" "k8s.io/helm/pkg/chartutil" - "k8s.io/helm/pkg/proto/hapi/chart" - "os" - "path" - "path/filepath" "sigs.k8s.io/yaml" ) type AppStoreDeploymentCommonService interface { - // move to DB service - GetInstalledAppByClusterNamespaceAndName(clusterId int, namespace string, appName string) (*appStoreBean.InstallAppVersionDTO, error) - GetInstalledAppByInstalledAppId(installedAppId int) (*appStoreBean.InstallAppVersionDTO, error) - - // common for all gitops? - CreateChartProxyAndGetPath(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*util.ChartCreateResponse, error) - - // keep here only - GetValuesAndRequirementGitConfig(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*util.ChartConfig, *util.ChartConfig, error) - GetGitCommitConfig(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, fileString string, filename string) (*util.ChartConfig, error) + // GetValuesString will return values string from the given valuesOverrideYaml GetValuesString(chartName, valuesOverrideYaml string) (string, error) + // GetRequirementsString will return requirement dependencies for the given appStoreVersionId GetRequirementsString(appStoreVersionId int) (string, error) - - // rethink - GenerateManifest(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (manifestResponse *AppStoreManifestResponse, err error) - - // gitops service - ParseGitRepoErrorResponse(err error) (bool, error) - CreateGitOpsRepoAndPushChart(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, builtChartPath string, requirementsConfig *util.ChartConfig, valuesConfig *util.ChartConfig) (*commonBean.ChartGitAttribute, bool, string, error) - GitOpsOperations(manifestResponse *AppStoreManifestResponse, installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*AppStoreGitOpsResponse, error) - GenerateManifestAndPerformGitOperations(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*AppStoreGitOpsResponse, error) + // CreateChartProxyAndGetPath parse chart in local directory and returns path of local dir and values.yaml + CreateChartProxyAndGetPath(chartCreateRequest *util.ChartCreateRequest) (*util.ChartCreateResponse, error) } type AppStoreDeploymentCommonServiceImpl struct { logger *zap.SugaredLogger - installedAppRepository repository.InstalledAppRepository appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository - environmentRepository repository2.EnvironmentRepository chartTemplateService util.ChartTemplateService - gitFactory *util.GitFactory - gitOpsConfigReadService config.GitOpsConfigReadService - gitOpsRemoteOperationService remote.GitOpsRemoteOperationService } func NewAppStoreDeploymentCommonServiceImpl( logger *zap.SugaredLogger, - installedAppRepository repository.InstalledAppRepository, appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository, - environmentRepository repository2.EnvironmentRepository, - chartTemplateService util.ChartTemplateService, - gitFactory *util.GitFactory, - gitOpsConfigReadService config.GitOpsConfigReadService, - gitOpsRemoteOperationService remote.GitOpsRemoteOperationService) *AppStoreDeploymentCommonServiceImpl { + chartTemplateService util.ChartTemplateService) *AppStoreDeploymentCommonServiceImpl { return &AppStoreDeploymentCommonServiceImpl{ logger: logger, - installedAppRepository: installedAppRepository, appStoreApplicationVersionRepository: appStoreApplicationVersionRepository, - environmentRepository: environmentRepository, chartTemplateService: chartTemplateService, - gitFactory: gitFactory, - gitOpsConfigReadService: gitOpsConfigReadService, - gitOpsRemoteOperationService: gitOpsRemoteOperationService, - } -} - -// TODO: move this from here - -func ParseChartCreateRequest(installAppRequestDTO *appStoreBean.InstallAppVersionDTO, chartPath string) *util.ChartCreateRequest { - return &util.ChartCreateRequest{ChartMetaData: &chart.Metadata{ - Name: installAppRequestDTO.AppName, - Version: "1.0.1", - }, ChartPath: chartPath} -} - -func ParseChartGitPushRequest(installAppRequestDTO *appStoreBean.InstallAppVersionDTO, repoURl string, tempRefChart string) *appStoreBean.PushChartToGitRequestDTO { - return &appStoreBean.PushChartToGitRequestDTO{ - AppName: installAppRequestDTO.AppName, - EnvName: installAppRequestDTO.Environment.Name, - ChartAppStoreName: installAppRequestDTO.AppStoreName, - RepoURL: repoURl, - TempChartRefDir: tempRefChart, - UserId: installAppRequestDTO.UserId, } } -type AppStoreManifestResponse struct { - ChartResponse *util.ChartCreateResponse - ValuesConfig *util.ChartConfig - RequirementsConfig *util.ChartConfig -} - -type AppStoreGitOpsResponse struct { - ChartGitAttribute *commonBean.ChartGitAttribute - GitHash string -} - -func (impl AppStoreDeploymentCommonServiceImpl) GetInstalledAppByClusterNamespaceAndName(clusterId int, namespace string, appName string) (*appStoreBean.InstallAppVersionDTO, error) { - installedApp, err := impl.installedAppRepository.GetInstalledApplicationByClusterIdAndNamespaceAndAppName(clusterId, namespace, appName) - if err != nil { - if err == pg.ErrNoRows { - impl.logger.Warnw("no installed apps found", "clusterId", clusterId) - return nil, nil - } else { - impl.logger.Errorw("error while fetching installed apps", "clusterId", clusterId, "error", err) - return nil, err - } - } - - if installedApp.Id > 0 { - installedAppVersion, err := impl.installedAppRepository.GetInstalledAppVersionByInstalledAppIdAndEnvId(installedApp.Id, installedApp.EnvironmentId) - if err != nil { - return nil, err - } - return adapter.GenerateInstallAppVersionDTO(installedApp, installedAppVersion), nil - } - - return nil, nil -} - -func (impl AppStoreDeploymentCommonServiceImpl) GetInstalledAppByInstalledAppId(installedAppId int) (*appStoreBean.InstallAppVersionDTO, error) { - installedAppVersion, err := impl.installedAppRepository.GetActiveInstalledAppVersionByInstalledAppId(installedAppId) - if err != nil { - return nil, err - } - installedApp := &installedAppVersion.InstalledApp - return adapter.GenerateInstallAppVersionDTO(installedApp, installedAppVersion), nil -} - func (impl AppStoreDeploymentCommonServiceImpl) GetValuesString(chartName, valuesOverrideYaml string) (string, error) { ValuesOverrideByte, err := yaml.YAMLToJSON([]byte(valuesOverrideYaml)) @@ -213,153 +106,17 @@ func (impl AppStoreDeploymentCommonServiceImpl) GetRequirementsString(appStoreVe return string(requirementDependenciesByte), nil } -func (impl AppStoreDeploymentCommonServiceImpl) GetGitCommitConfig(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, fileString string, filename string) (*util.ChartConfig, error) { - appStoreAppVersion, err := impl.appStoreApplicationVersionRepository.FindById(installAppVersionRequest.AppStoreVersion) - if err != nil { - impl.logger.Errorw("fetching error", "err", err) - return nil, err - } - environment, err := impl.environmentRepository.FindById(installAppVersionRequest.EnvironmentId) - if err != nil { - impl.logger.Errorw("fetching error", "err", err) - return nil, err - } - - argocdAppName := installAppVersionRequest.AppName + "-" + environment.Name - gitOpsRepoName := impl.gitOpsConfigReadService.GetGitOpsRepoName(installAppVersionRequest.AppName) - userEmailId, userName := impl.gitOpsConfigReadService.GetUserEmailIdAndNameForGitOpsCommit(installAppVersionRequest.UserId) - YamlConfig := &util.ChartConfig{ - FileName: filename, - FileContent: fileString, - ChartName: installAppVersionRequest.AppName, - ChartLocation: argocdAppName, - ChartRepoName: gitOpsRepoName, - ReleaseMessage: fmt.Sprintf("release-%d-env-%d ", appStoreAppVersion.Id, environment.Id), - UserEmailId: userEmailId, - UserName: userName, - } - return YamlConfig, nil -} - -func (impl AppStoreDeploymentCommonServiceImpl) GetValuesAndRequirementGitConfig(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*util.ChartConfig, *util.ChartConfig, error) { - - appStoreAppVersion, err := impl.appStoreApplicationVersionRepository.FindById(installAppVersionRequest.AppStoreVersion) - if err != nil { - impl.logger.Errorw("fetching error", "err", err) - return nil, nil, err - } - values, err := impl.GetValuesString(appStoreAppVersion.AppStore.Name, installAppVersionRequest.ValuesOverrideYaml) - if err != nil { - impl.logger.Errorw("error in getting values fot installedAppVersionRequest", "err", err) - return nil, nil, err - } - dependency, err := impl.GetRequirementsString(installAppVersionRequest.AppStoreVersion) - if err != nil { - impl.logger.Errorw("error in getting dependency array fot installedAppVersionRequest", "err", err) - return nil, nil, err - } - valuesConfig, err := impl.GetGitCommitConfig(installAppVersionRequest, values, appStoreBean.VALUES_YAML_FILE) - if err != nil { - impl.logger.Errorw("error in creating values config for git", "err", err) - return nil, nil, err - } - RequirementConfig, err := impl.GetGitCommitConfig(installAppVersionRequest, dependency, appStoreBean.REQUIREMENTS_YAML_FILE) - if err != nil { - impl.logger.Errorw("error in creating dependency config for git", "err", err) - return nil, nil, err - } - return valuesConfig, RequirementConfig, nil -} - -func (impl AppStoreDeploymentCommonServiceImpl) GenerateManifest(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (manifestResponse *AppStoreManifestResponse, err error) { - - manifestResponse = &AppStoreManifestResponse{} - - ChartCreateResponse, err := impl.CreateChartProxyAndGetPath(installAppVersionRequest) - if err != nil { - impl.logger.Errorw("Error in building chart while generating manifest", "err", err) - return manifestResponse, err - } - valuesConfig, dependencyConfig, err := impl.GetValuesAndRequirementGitConfig(installAppVersionRequest) - if err != nil { - impl.logger.Errorw("error in fetching values and requirements.yaml config while generating manifest", "err", err) - return manifestResponse, err - } - - manifestResponse.ChartResponse = ChartCreateResponse - manifestResponse.ValuesConfig = valuesConfig - manifestResponse.RequirementsConfig = dependencyConfig - - return manifestResponse, nil -} - -// CreateGitOpsRepo creates a gitOps repo with readme -func (impl AppStoreDeploymentCommonServiceImpl) CreateGitOpsRepo(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (string, bool, error) { - if len(installAppVersionRequest.GitOpsRepoName) == 0 { - //here gitops repo will be the app name, to breaking the mono repo structure - gitOpsRepoName := impl.gitOpsConfigReadService.GetGitOpsRepoName(installAppVersionRequest.AppName) - installAppVersionRequest.GitOpsRepoName = gitOpsRepoName - } - bitbucketMetadata, err := impl.gitOpsConfigReadService.GetBitbucketMetadata() - if err != nil { - impl.logger.Errorw("error in getting bitbucket metadata", "err", err) - return "", false, err - } - //getting user name & emailId for commit author data - gitRepoRequest := &bean.GitOpsConfigDto{ - GitRepoName: installAppVersionRequest.GitOpsRepoName, - Description: "helm chart for " + installAppVersionRequest.GitOpsRepoName, - BitBucketWorkspaceId: bitbucketMetadata.BitBucketWorkspaceId, - BitBucketProjectKey: bitbucketMetadata.BitBucketProjectKey, - } - repoUrl, isNew, err := impl.gitOpsRemoteOperationService.CreateRepository(gitRepoRequest, installAppVersionRequest.UserId) - if err != nil { - impl.logger.Errorw("error in creating git project", "name", installAppVersionRequest.GitOpsRepoName, "err", err) - return "", false, err - } - return repoUrl, isNew, err -} - -// AddConfigFileToChart will override requirements.yaml file in chart -func (impl AppStoreDeploymentCommonServiceImpl) AddConfigFileToChart(config *util.ChartConfig, dir string, clonedDir string) error { - filePath := filepath.Join(clonedDir, config.FileName) - file, err := os.Create(filePath) - if err != nil { - impl.logger.Errorw("error in creating file", "err", err, "fileName", config.FileName) - return err - } - defer file.Close() - _, err = file.Write([]byte(config.FileContent)) - if err != nil { - impl.logger.Errorw("error in writing file content", "err", err, "fileName", config.FileName) - return err - } - destinationFilePath := filepath.Join(dir, config.FileName) - err = util2.MoveFileToDestination(filePath, destinationFilePath) - if err != nil { - impl.logger.Errorw("error in moving file from source to destination", "err", err) - return err - } - return nil -} - -// CreateChartProxyAndGetPath parse chart in local directory and returns path of local dir and values.yaml -func (impl AppStoreDeploymentCommonServiceImpl) CreateChartProxyAndGetPath(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*util.ChartCreateResponse, error) { - +func (impl AppStoreDeploymentCommonServiceImpl) CreateChartProxyAndGetPath(chartCreateRequest *util.ChartCreateRequest) (*util.ChartCreateResponse, error) { ChartCreateResponse := &util.ChartCreateResponse{} - template := appStoreBean.CHART_PROXY_TEMPLATE - chartPath := path.Join(appStoreBean.RefChartProxyDirPath, template) - valid, err := chartutil.IsChartDir(chartPath) + valid, err := chartutil.IsChartDir(chartCreateRequest.ChartPath) if err != nil || !valid { - impl.logger.Errorw("invalid base chart", "dir", chartPath, "err", err) + impl.logger.Errorw("invalid base chart", "dir", chartCreateRequest.ChartPath, "err", err) return ChartCreateResponse, err } - chartCreateRequest := ParseChartCreateRequest(installAppVersionRequest, chartPath) chartCreateResponse, err := impl.chartTemplateService.BuildChartProxyForHelmApps(chartCreateRequest) if err != nil { impl.logger.Errorw("Error in building chart proxy", "err", err) return chartCreateResponse, err } return chartCreateResponse, nil - } diff --git a/pkg/appStore/deployment/common/AppStoreDeploymentGitService.go b/pkg/appStore/deployment/common/AppStoreDeploymentGitService.go deleted file mode 100644 index 9b9846f415..0000000000 --- a/pkg/appStore/deployment/common/AppStoreDeploymentGitService.go +++ /dev/null @@ -1,184 +0,0 @@ -package appStoreDeploymentCommon - -import ( - "fmt" - "github.com/devtron-labs/devtron/internal/util" - "github.com/devtron-labs/devtron/pkg/appStore/bean" - commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" - "github.com/google/go-github/github" - "github.com/microsoft/azure-devops-go-api/azuredevops" - dirCopy "github.com/otiai10/copy" - "github.com/xanzy/go-gitlab" - "net/http" - "os" - "path/filepath" - "regexp" -) - -func (impl AppStoreDeploymentCommonServiceImpl) ParseGitRepoErrorResponse(err error) (bool, error) { - //update values yaml in chart - noTargetFound := false - if err != nil { - if errorResponse, ok := err.(*github.ErrorResponse); ok && errorResponse.Response.StatusCode == http.StatusNotFound { - impl.logger.Errorw("no content found while updating git repo on github, do auto fix", "error", err) - noTargetFound = true - } - if errorResponse, ok := err.(azuredevops.WrappedError); ok && *errorResponse.StatusCode == http.StatusNotFound { - impl.logger.Errorw("no content found while updating git repo on azure, do auto fix", "error", err) - noTargetFound = true - } - if errorResponse, ok := err.(*azuredevops.WrappedError); ok && *errorResponse.StatusCode == http.StatusNotFound { - impl.logger.Errorw("no content found while updating git repo on azure, do auto fix", "error", err) - noTargetFound = true - } - if errorResponse, ok := err.(*gitlab.ErrorResponse); ok && errorResponse.Response.StatusCode == http.StatusNotFound { - impl.logger.Errorw("no content found while updating git repo gitlab, do auto fix", "error", err) - noTargetFound = true - } - if err.Error() == util.BITBUCKET_REPO_NOT_FOUND_ERROR { - impl.logger.Errorw("no content found while updating git repo bitbucket, do auto fix", "error", err) - noTargetFound = true - } - } - return noTargetFound, err -} - -// CreateGitOpsRepoAndPushChart is a wrapper for creating GitOps repo and pushing chart to created repo -func (impl AppStoreDeploymentCommonServiceImpl) CreateGitOpsRepoAndPushChart(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, builtChartPath string, requirementsConfig *util.ChartConfig, valuesConfig *util.ChartConfig) (*commonBean.ChartGitAttribute, bool, string, error) { - repoURL, isNew, err := impl.CreateGitOpsRepo(installAppVersionRequest) - if err != nil { - impl.logger.Errorw("Error in creating gitops repo for ", "appName", installAppVersionRequest.AppName, "err", err) - return nil, false, "", err - } - pushChartToGitRequest := ParseChartGitPushRequest(installAppVersionRequest, repoURL, builtChartPath) - chartGitAttribute, commitHash, err := impl.pushChartToGitOpsRepo(pushChartToGitRequest, requirementsConfig, valuesConfig) - if err != nil { - impl.logger.Errorw("error in pushing chart to git", "err", err) - return nil, false, "", err - } - return chartGitAttribute, isNew, commitHash, err -} - -func (impl AppStoreDeploymentCommonServiceImpl) GitOpsOperations(manifestResponse *AppStoreManifestResponse, installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*AppStoreGitOpsResponse, error) { - appStoreGitOpsResponse := &AppStoreGitOpsResponse{} - chartGitAttribute, isNew, githash, err := impl.CreateGitOpsRepoAndPushChart(installAppVersionRequest, manifestResponse.ChartResponse.BuiltChartPath, manifestResponse.RequirementsConfig, manifestResponse.ValuesConfig) - if err != nil { - impl.logger.Errorw("Error in pushing chart to git", "err", err) - return appStoreGitOpsResponse, err - } - space := regexp.MustCompile(`\s+`) - appStoreName := space.ReplaceAllString(installAppVersionRequest.AppName, "-") - clonedDir := impl.gitFactory.GitWorkingDir + "" + appStoreName - - // Checking this is the first time chart has been pushed , if yes requirements.yaml has been already pushed with chart as there was sync-delay with github api. - // step-2 commit dependencies and values in git - if !isNew { - _, _, err = impl.gitOpsRemoteOperationService.CommitValues(manifestResponse.RequirementsConfig) - if err != nil { - impl.logger.Errorw("error in committing dependency config to git", "err", err) - return appStoreGitOpsResponse, err - } - err = impl.gitOpsRemoteOperationService.GitPull(clonedDir, chartGitAttribute.RepoUrl, appStoreName) - if err != nil { - impl.logger.Errorw("error in git pull", "err", err) - return appStoreGitOpsResponse, err - } - - githash, _, err = impl.gitOpsRemoteOperationService.CommitValues(manifestResponse.ValuesConfig) - if err != nil { - impl.logger.Errorw("error in committing values config to git", "err", err) - return appStoreGitOpsResponse, err - } - err = impl.gitOpsRemoteOperationService.GitPull(clonedDir, chartGitAttribute.RepoUrl, appStoreName) - if err != nil { - impl.logger.Errorw("error in git pull", "err", err) - return appStoreGitOpsResponse, err - } - } - appStoreGitOpsResponse.ChartGitAttribute = chartGitAttribute - appStoreGitOpsResponse.GitHash = githash - return appStoreGitOpsResponse, nil -} - -func (impl AppStoreDeploymentCommonServiceImpl) GenerateManifestAndPerformGitOperations(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*AppStoreGitOpsResponse, error) { - appStoreGitOpsResponse := &AppStoreGitOpsResponse{} - manifest, err := impl.GenerateManifest(installAppVersionRequest) - if err != nil { - impl.logger.Errorw("error in performing manifest and git operations", "err", err) - return nil, err - } - gitOpsResponse, err := impl.GitOpsOperations(manifest, installAppVersionRequest) - if err != nil { - impl.logger.Errorw("error in performing gitops operation", "err", err) - return nil, err - } - installAppVersionRequest.GitHash = gitOpsResponse.GitHash - appStoreGitOpsResponse.ChartGitAttribute = gitOpsResponse.ChartGitAttribute - return appStoreGitOpsResponse, nil -} - -// pushChartToGitOpsRepo pushes built chart to GitOps repo -func (impl AppStoreDeploymentCommonServiceImpl) pushChartToGitOpsRepo(PushChartToGitRequest *appStoreBean.PushChartToGitRequestDTO, requirementsConfig *util.ChartConfig, valuesConfig *util.ChartConfig) (*commonBean.ChartGitAttribute, string, error) { - space := regexp.MustCompile(`\s+`) - appStoreName := space.ReplaceAllString(PushChartToGitRequest.ChartAppStoreName, "-") - chartDir := fmt.Sprintf("%s-%s", PushChartToGitRequest.AppName, impl.chartTemplateService.GetDir()) - clonedDir := impl.gitFactory.GitService.GetCloneDirectory(chartDir) - if _, err := os.Stat(clonedDir); os.IsNotExist(err) { - clonedDir, err = impl.gitFactory.GitService.Clone(PushChartToGitRequest.RepoURL, chartDir) - if err != nil { - impl.logger.Errorw("error in cloning repo", "url", PushChartToGitRequest.RepoURL, "err", err) - return nil, "", err - } - } else { - err = impl.gitOpsRemoteOperationService.GitPull(clonedDir, PushChartToGitRequest.RepoURL, appStoreName) - if err != nil { - return nil, "", err - } - } - acdAppName := fmt.Sprintf("%s-%s", PushChartToGitRequest.AppName, PushChartToGitRequest.EnvName) - dir := filepath.Join(clonedDir, acdAppName) - err := os.MkdirAll(dir, os.ModePerm) - if err != nil { - impl.logger.Errorw("error in making dir", "err", err) - return nil, "", err - } - err = dirCopy.Copy(PushChartToGitRequest.TempChartRefDir, dir) - if err != nil { - impl.logger.Errorw("error copying dir", "err", err) - return nil, "", err - } - err = impl.AddConfigFileToChart(requirementsConfig, dir, clonedDir) - if err != nil { - impl.logger.Errorw("error in adding requirements.yaml to chart", "err", err, "appName", PushChartToGitRequest.AppName) - return nil, "", err - } - err = impl.AddConfigFileToChart(valuesConfig, dir, clonedDir) - if err != nil { - impl.logger.Errorw("error in adding values.yaml to chart", "err", err, "appName", PushChartToGitRequest.AppName) - return nil, "", err - } - userEmailId, userName := impl.gitOpsConfigReadService.GetUserEmailIdAndNameForGitOpsCommit(PushChartToGitRequest.UserId) - commit, err := impl.gitFactory.GitService.CommitAndPushAllChanges(clonedDir, "first commit", userName, userEmailId) - if err != nil { - impl.logger.Errorw("error in pushing git", "err", err) - impl.logger.Warn("re-trying, taking pull and then push again") - err = impl.gitOpsRemoteOperationService.GitPull(clonedDir, PushChartToGitRequest.RepoURL, acdAppName) - if err != nil { - impl.logger.Errorw("error in git pull", "err", err, "appName", acdAppName) - return nil, "", err - } - err = dirCopy.Copy(PushChartToGitRequest.TempChartRefDir, dir) - if err != nil { - impl.logger.Errorw("error copying dir", "err", err) - return nil, "", err - } - commit, err = impl.gitFactory.GitService.CommitAndPushAllChanges(clonedDir, "first commit", userName, userEmailId) - if err != nil { - impl.logger.Errorw("error in pushing git", "err", err) - return nil, "", err - } - } - impl.logger.Debugw("template committed", "url", PushChartToGitRequest.RepoURL, "commit", commit) - defer impl.chartTemplateService.CleanDir(clonedDir) - return &commonBean.ChartGitAttribute{RepoUrl: PushChartToGitRequest.RepoURL, ChartLocation: acdAppName}, commit, err -} diff --git a/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go b/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go deleted file mode 100644 index 87f3bbd0a4..0000000000 --- a/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go +++ /dev/null @@ -1,289 +0,0 @@ -/* - * Copyright (c) 2020 Devtron Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package appStoreDeploymentFullMode - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "github.com/devtron-labs/common-lib/pubsub-lib" - "github.com/devtron-labs/common-lib/pubsub-lib/model" - commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" - "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" - "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote" - "time" - - "github.com/devtron-labs/devtron/client/argocdServer" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/devtron-labs/devtron/pkg/app/status" - appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" - appStoreDeploymentCommon "github.com/devtron-labs/devtron/pkg/appStore/deployment/common" - repository4 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" - appStoreDiscoverRepository "github.com/devtron-labs/devtron/pkg/appStore/discover/repository" - repository5 "github.com/devtron-labs/devtron/pkg/cluster/repository" - "github.com/devtron-labs/devtron/pkg/sql" - util2 "github.com/devtron-labs/devtron/pkg/util" - "github.com/devtron-labs/devtron/util/argo" - "github.com/go-pg/pg" - - "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" - application2 "github.com/devtron-labs/devtron/client/argocdServer/application" - "go.uber.org/zap" -) - -const ( - DEFAULT_ENVIRONMENT_OR_NAMESPACE_OR_PROJECT = "devtron" - CLUSTER_COMPONENT_DIR_PATH = "/cluster/component" -) - -// ACD operation and git operation -type AppStoreDeploymentFullModeService interface { - AppStoreDeployOperationACD(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, chartGitAttr *commonBean.ChartGitAttribute, ctx context.Context, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) - UpdateValuesYaml(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) - UpdateRequirementYaml(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, appStoreAppVersion *appStoreDiscoverRepository.AppStoreApplicationVersion) error - GetGitOpsRepoName(appName string, environmentName string) (string, error) -} - -type AppStoreDeploymentFullModeServiceImpl struct { - logger *zap.SugaredLogger - acdClient application2.ServiceClient - ArgoK8sClient argocdServer.ArgoK8sClient - aCDAuthConfig *util2.ACDAuthConfig - argoUserService argo.ArgoUserService - pipelineStatusTimelineService status.PipelineStatusTimelineService - appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService - argoClientWrapperService argocdServer.ArgoClientWrapperService - pubSubClient *pubsub_lib.PubSubClientServiceImpl - installedAppRepositoryHistory repository4.InstalledAppVersionHistoryRepository - ACDConfig *argocdServer.ACDConfig - gitOpsConfigReadService config.GitOpsConfigReadService - gitOpsRemoteOperationService remote.GitOpsRemoteOperationService -} - -func NewAppStoreDeploymentFullModeServiceImpl(logger *zap.SugaredLogger, - acdClient application2.ServiceClient, - argoK8sClient argocdServer.ArgoK8sClient, aCDAuthConfig *util2.ACDAuthConfig, - argoUserService argo.ArgoUserService, pipelineStatusTimelineService status.PipelineStatusTimelineService, - appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService, - argoClientWrapperService argocdServer.ArgoClientWrapperService, - pubSubClient *pubsub_lib.PubSubClientServiceImpl, - installedAppRepositoryHistory repository4.InstalledAppVersionHistoryRepository, - ACDConfig *argocdServer.ACDConfig, - gitOpsConfigReadService config.GitOpsConfigReadService, - gitOpsRemoteOperationService remote.GitOpsRemoteOperationService) *AppStoreDeploymentFullModeServiceImpl { - appStoreDeploymentFullModeServiceImpl := &AppStoreDeploymentFullModeServiceImpl{ - logger: logger, - acdClient: acdClient, - ArgoK8sClient: argoK8sClient, - aCDAuthConfig: aCDAuthConfig, - argoUserService: argoUserService, - pipelineStatusTimelineService: pipelineStatusTimelineService, - appStoreDeploymentCommonService: appStoreDeploymentCommonService, - argoClientWrapperService: argoClientWrapperService, - pubSubClient: pubSubClient, - installedAppRepositoryHistory: installedAppRepositoryHistory, - ACDConfig: ACDConfig, - gitOpsConfigReadService: gitOpsConfigReadService, - gitOpsRemoteOperationService: gitOpsRemoteOperationService, - } - err := appStoreDeploymentFullModeServiceImpl.subscribeHelmInstallStatus() - if err != nil { - return nil - } - return appStoreDeploymentFullModeServiceImpl -} - -func (impl AppStoreDeploymentFullModeServiceImpl) AppStoreDeployOperationACD(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, chartGitAttr *commonBean.ChartGitAttribute, ctx context.Context, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) { - ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) - defer cancel() - //STEP 4: registerInArgo - err := impl.argoClientWrapperService.RegisterGitOpsRepoInArgo(ctx, chartGitAttr.RepoUrl) - if err != nil { - impl.logger.Errorw("error in argo registry", "err", err) - return nil, err - } - //STEP 5: createInArgo - err = impl.createInArgo(chartGitAttr, ctx, *installAppVersionRequest.Environment, installAppVersionRequest.ACDAppName) - if err != nil { - impl.logger.Errorw("error in create in argo", "err", err) - return nil, err - } - //STEP 6: Force Sync ACD - works like trigger deployment - //impl.SyncACD(installAppVersionRequest.ACDAppName, ctx) - - //STEP 7: normal refresh ACD - update for step 6 to avoid delay - syncTime := time.Now() - err = impl.argoClientWrapperService.SyncArgoCDApplicationIfNeededAndRefresh(ctx, installAppVersionRequest.ACDAppName) - if err != nil { - impl.logger.Errorw("error in getting the argo application with normal refresh", "err", err) - return nil, err - } - if !impl.ACDConfig.ArgoCDAutoSyncEnabled { - timeline := &pipelineConfig.PipelineStatusTimeline{ - InstalledAppVersionHistoryId: installAppVersionRequest.InstalledAppVersionHistoryId, - Status: pipelineConfig.TIMELINE_STATUS_ARGOCD_SYNC_COMPLETED, - StatusDetail: "argocd sync completed.", - StatusTime: syncTime, - AuditLog: sql.AuditLog{ - CreatedBy: installAppVersionRequest.UserId, - CreatedOn: time.Now(), - UpdatedBy: installAppVersionRequest.UserId, - UpdatedOn: time.Now(), - }, - } - err = impl.pipelineStatusTimelineService.SaveTimeline(timeline, tx, true) - if err != nil { - impl.logger.Errorw("error in creating timeline for argocd sync", "err", err, "timeline", timeline) - } - } - - return installAppVersionRequest, nil -} - -func (impl AppStoreDeploymentFullModeServiceImpl) createInArgo(chartGitAttribute *commonBean.ChartGitAttribute, ctx context.Context, envModel repository5.Environment, argocdAppName string) error { - appNamespace := envModel.Namespace - if appNamespace == "" { - appNamespace = "default" - } - appreq := &argocdServer.AppTemplate{ - ApplicationName: argocdAppName, - Namespace: impl.aCDAuthConfig.ACDConfigMapNamespace, - TargetNamespace: appNamespace, - TargetServer: envModel.Cluster.ServerUrl, - Project: "default", - ValuesFile: fmt.Sprintf("values.yaml"), - RepoPath: chartGitAttribute.ChartLocation, - RepoUrl: chartGitAttribute.RepoUrl, - AutoSyncEnabled: impl.ACDConfig.ArgoCDAutoSyncEnabled, - } - _, err := impl.ArgoK8sClient.CreateAcdApp(appreq, envModel.Cluster, argocdServer.ARGOCD_APPLICATION_TEMPLATE) - //create - if err != nil { - impl.logger.Errorw("error in creating argo cd app ", "err", err) - return err - } - return nil -} - -func (impl AppStoreDeploymentFullModeServiceImpl) GetGitOpsRepoName(appName string, environmentName string) (string, error) { - gitOpsRepoName := "" - //this method should only call in case of argo-integration and gitops configured - acdToken, err := impl.argoUserService.GetLatestDevtronArgoCdUserToken() - if err != nil { - impl.logger.Errorw("error in getting acd token", "err", err) - return "", err - } - ctx := context.Background() - ctx = context.WithValue(ctx, "token", acdToken) - acdAppName := fmt.Sprintf("%s-%s", appName, environmentName) - application, err := impl.acdClient.Get(ctx, &application.ApplicationQuery{Name: &acdAppName}) - if err != nil { - impl.logger.Errorw("no argo app exists", "acdAppName", acdAppName, "err", err) - return "", err - } - if application != nil { - gitOpsRepoUrl := application.Spec.Source.RepoURL - gitOpsRepoName = impl.gitOpsConfigReadService.GetGitOpsRepoNameFromUrl(gitOpsRepoUrl) - } - return gitOpsRepoName, nil -} - -func (impl AppStoreDeploymentFullModeServiceImpl) UpdateValuesYaml(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) { - - valuesString, err := impl.appStoreDeploymentCommonService.GetValuesString(installAppVersionRequest.AppStoreName, installAppVersionRequest.ValuesOverrideYaml) - if err != nil { - impl.logger.Errorw("error in getting values string", "err", err) - return nil, err - } - - valuesGitConfig, err := impl.appStoreDeploymentCommonService.GetGitCommitConfig(installAppVersionRequest, valuesString, appStoreBean.VALUES_YAML_FILE) - if err != nil { - impl.logger.Errorw("error in getting git commit config", "err", err) - } - - commitHash, _, err := impl.gitOpsRemoteOperationService.CommitValues(valuesGitConfig) - if err != nil { - impl.logger.Errorw("error in git commit", "err", err) - return installAppVersionRequest, errors.New(pipelineConfig.TIMELINE_STATUS_GIT_COMMIT_FAILED) - } - //update timeline status for git commit state - installAppVersionRequest.GitHash = commitHash - return installAppVersionRequest, nil -} - -func (impl AppStoreDeploymentFullModeServiceImpl) UpdateRequirementYaml(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, appStoreAppVersion *appStoreDiscoverRepository.AppStoreApplicationVersion) error { - - requirementsString, err := impl.appStoreDeploymentCommonService.GetRequirementsString(appStoreAppVersion.Id) - if err != nil { - impl.logger.Errorw("error in getting requirements string", "err", err) - return err - } - - requirementsGitConfig, err := impl.appStoreDeploymentCommonService.GetGitCommitConfig(installAppVersionRequest, requirementsString, appStoreBean.REQUIREMENTS_YAML_FILE) - if err != nil { - impl.logger.Errorw("error in getting git commit config", "err", err) - return err - } - - _, _, err = impl.gitOpsRemoteOperationService.CommitValues(requirementsGitConfig) - if err != nil { - impl.logger.Errorw("error in values commit", "err", err) - return errors.New(pipelineConfig.TIMELINE_STATUS_GIT_COMMIT_FAILED) - } - - return nil -} - -func (impl AppStoreDeploymentFullModeServiceImpl) subscribeHelmInstallStatus() error { - - callback := func(msg *model.PubSubMsg) { - - impl.logger.Debug("received helm install status event - HELM_INSTALL_STATUS", "data", msg.Data) - helmInstallNatsMessage := &appStoreBean.HelmReleaseStatusConfig{} - err := json.Unmarshal([]byte(msg.Data), helmInstallNatsMessage) - if err != nil { - impl.logger.Errorw("error in unmarshalling helm install status nats message", "err", err) - return - } - - installedAppVersionHistory, err := impl.installedAppRepositoryHistory.GetInstalledAppVersionHistory(helmInstallNatsMessage.InstallAppVersionHistoryId) - if err != nil { - impl.logger.Errorw("error in fetching installed app by installed app id in subscribe helm status callback", "err", err) - return - } - if helmInstallNatsMessage.ErrorInInstallation { - installedAppVersionHistory.Status = pipelineConfig.WorkflowFailed - } else { - installedAppVersionHistory.Status = pipelineConfig.WorkflowSucceeded - } - installedAppVersionHistory.HelmReleaseStatusConfig = msg.Data - _, err = impl.installedAppRepositoryHistory.UpdateInstalledAppVersionHistory(installedAppVersionHistory, nil) - if err != nil { - impl.logger.Errorw("error in updating helm release status data in installedAppVersionHistoryRepository", "err", err) - return - } - } - - err := impl.pubSubClient.Subscribe(pubsub_lib.HELM_CHART_INSTALL_STATUS_TOPIC, callback) - if err != nil { - impl.logger.Error(err) - return err - } - return nil -} diff --git a/pkg/appStore/deployment/service/AppStoreDeploymentDBService.go b/pkg/appStore/deployment/service/AppStoreDeploymentDBService.go index dea7c70d0b..72c404e27d 100644 --- a/pkg/appStore/deployment/service/AppStoreDeploymentDBService.go +++ b/pkg/appStore/deployment/service/AppStoreDeploymentDBService.go @@ -178,3 +178,35 @@ func (impl AppStoreDeploymentServiceImpl) AppStoreDeployOperationStatusUpdate(in } return true, nil } + +func (impl AppStoreDeploymentServiceImpl) GetInstalledAppByClusterNamespaceAndName(clusterId int, namespace string, appName string) (*appStoreBean.InstallAppVersionDTO, error) { + installedApp, err := impl.installedAppRepository.GetInstalledApplicationByClusterIdAndNamespaceAndAppName(clusterId, namespace, appName) + if err != nil { + if err == pg.ErrNoRows { + impl.logger.Warnw("no installed apps found", "clusterId", clusterId) + return nil, nil + } else { + impl.logger.Errorw("error while fetching installed apps", "clusterId", clusterId, "error", err) + return nil, err + } + } + + if installedApp.Id > 0 { + installedAppVersion, err := impl.installedAppRepository.GetInstalledAppVersionByInstalledAppIdAndEnvId(installedApp.Id, installedApp.EnvironmentId) + if err != nil { + return nil, err + } + return adapter.GenerateInstallAppVersionDTO(installedApp, installedAppVersion), nil + } + + return nil, nil +} + +func (impl AppStoreDeploymentServiceImpl) GetInstalledAppByInstalledAppId(installedAppId int) (*appStoreBean.InstallAppVersionDTO, error) { + installedAppVersion, err := impl.installedAppRepository.GetActiveInstalledAppVersionByInstalledAppId(installedAppId) + if err != nil { + return nil, err + } + installedApp := &installedAppVersion.InstalledApp + return adapter.GenerateInstallAppVersionDTO(installedApp, installedAppVersion), nil +} diff --git a/pkg/appStore/deployment/service/AppStoreDeploymentService.go b/pkg/appStore/deployment/service/AppStoreDeploymentService.go index 1fbb14a37c..a515c55594 100644 --- a/pkg/appStore/deployment/service/AppStoreDeploymentService.go +++ b/pkg/appStore/deployment/service/AppStoreDeploymentService.go @@ -23,8 +23,10 @@ import ( "errors" "fmt" "github.com/caarlos0/env/v6" - client "github.com/devtron-labs/devtron/api/helm-app" + bean3 "github.com/devtron-labs/devtron/api/helm-app/bean" + bean4 "github.com/devtron-labs/devtron/api/helm-app/gRPC" openapi "github.com/devtron-labs/devtron/api/helm-app/openapiClient" + "github.com/devtron-labs/devtron/api/helm-app/service" openapi2 "github.com/devtron-labs/devtron/api/openapi/openapiClient" "github.com/devtron-labs/devtron/client/argocdServer" "github.com/devtron-labs/devtron/internal/constants" @@ -39,6 +41,7 @@ import ( appStoreDeploymentCommon "github.com/devtron-labs/devtron/pkg/appStore/deployment/common" "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" appStoreDeploymentTool "github.com/devtron-labs/devtron/pkg/appStore/deployment/tool" + bean2 "github.com/devtron-labs/devtron/pkg/appStore/deployment/tool/bean" appStoreDiscoverRepository "github.com/devtron-labs/devtron/pkg/appStore/discover/repository" "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/cluster" @@ -64,10 +67,10 @@ type AppStoreDeploymentService interface { GetInstalledApp(id int) (*appStoreBean.InstallAppVersionDTO, error) GetAllInstalledAppsByAppStoreId(w http.ResponseWriter, r *http.Request, token string, appStoreId int) ([]appStoreBean.InstalledAppsResponse, error) DeleteInstalledApp(ctx context.Context, installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*appStoreBean.InstallAppVersionDTO, error) - LinkHelmApplicationToChartStore(ctx context.Context, request *openapi.UpdateReleaseWithChartLinkingRequest, appIdentifier *client.AppIdentifier, userId int32) (*openapi.UpdateReleaseResponse, bool, error) + LinkHelmApplicationToChartStore(ctx context.Context, request *openapi.UpdateReleaseWithChartLinkingRequest, appIdentifier *service.AppIdentifier, userId int32) (*openapi.UpdateReleaseResponse, bool, error) RollbackApplication(ctx context.Context, request *openapi2.RollbackReleaseRequest, installedApp *appStoreBean.InstallAppVersionDTO, userId int32) (bool, error) UpdateInstallAppVersionHistory(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*repository.InstalledAppVersionHistory, error) - GetDeploymentHistory(ctx context.Context, installedApp *appStoreBean.InstallAppVersionDTO) (*client.DeploymentHistoryAndInstalledAppInfo, error) + GetDeploymentHistory(ctx context.Context, installedApp *appStoreBean.InstallAppVersionDTO) (*bean3.DeploymentHistoryAndInstalledAppInfo, error) GetDeploymentHistoryInfo(ctx context.Context, installedApp *appStoreBean.InstallAppVersionDTO, installedAppVersionHistoryId int) (*openapi.HelmAppDeploymentManifestDetail, error) UpdateInstalledApp(ctx context.Context, installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*appStoreBean.InstallAppVersionDTO, error) UpdateInstalledAppVersionHistoryWithGitHash(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, tx *pg.Tx) error @@ -76,6 +79,9 @@ type AppStoreDeploymentService interface { UpdateProjectHelmApp(updateAppRequest *appStoreBean.UpdateProjectHelmAppDTO) error UpdatePreviousDeploymentStatusForAppStore(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, triggeredAt time.Time, err error) error UpdateInstallAppVersionHistoryStatus(installedAppVersionHistoryId int, status string) error + + GetInstalledAppByClusterNamespaceAndName(clusterId int, namespace string, appName string) (*appStoreBean.InstallAppVersionDTO, error) + GetInstalledAppByInstalledAppId(installedAppId int) (*appStoreBean.InstallAppVersionDTO, error) } type DeploymentServiceTypeConfig struct { @@ -101,7 +107,7 @@ type AppStoreDeploymentServiceImpl struct { appStoreDeploymentArgoCdService appStoreDeploymentTool.AppStoreDeploymentArgoCdService environmentService cluster.EnvironmentService clusterService cluster.ClusterService - helmAppService client.HelmAppService + helmAppService service.HelmAppService appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService installedAppRepositoryHistory repository.InstalledAppVersionHistoryRepository gitOpsRepository repository2.GitOpsConfigRepository @@ -116,7 +122,7 @@ func NewAppStoreDeploymentServiceImpl(logger *zap.SugaredLogger, installedAppRep clusterInstalledAppsRepository repository.ClusterInstalledAppsRepository, appRepository app.AppRepository, appStoreDeploymentHelmService appStoreDeploymentTool.AppStoreDeploymentHelmService, appStoreDeploymentArgoCdService appStoreDeploymentTool.AppStoreDeploymentArgoCdService, environmentService cluster.EnvironmentService, - clusterService cluster.ClusterService, helmAppService client.HelmAppService, appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService, + clusterService cluster.ClusterService, helmAppService service.HelmAppService, appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService, installedAppRepositoryHistory repository.InstalledAppVersionHistoryRepository, gitOpsRepository repository2.GitOpsConfigRepository, deploymentTypeConfig *DeploymentServiceTypeConfig, aCDConfig *argocdServer.ACDConfig, gitOpsConfigReadService config.GitOpsConfigReadService, @@ -181,19 +187,18 @@ func (impl AppStoreDeploymentServiceImpl) InstallApp(installAppVersionRequest *a _ = impl.appStoreDeploymentArgoCdService.SaveTimelineForACDHelmApps(installAppVersionRequest, pipelineConfig.TIMELINE_STATUS_DEPLOYMENT_INITIATED, "Deployment initiated successfully.", time.Now(), tx) } - manifest, err := impl.appStoreDeploymentCommonService.GenerateManifest(installAppVersionRequest) - if err != nil { - impl.logger.Errorw("error in performing manifest and git operations", "err", err) - return nil, err - } - - if installAppVersionRequest.DeploymentAppType == util.PIPELINE_DEPLOYMENT_TYPE_MANIFEST_DOWNLOAD { + if util.IsManifestDownload(installAppVersionRequest.DeploymentAppType) { _ = impl.appStoreDeploymentArgoCdService.SaveTimelineForACDHelmApps(installAppVersionRequest, pipelineConfig.TIMELINE_DESCRIPTION_MANIFEST_GENERATED, "Manifest generated successfully.", time.Now(), tx) } - var gitOpsResponse *appStoreDeploymentCommon.AppStoreGitOpsResponse + var gitOpsResponse *bean2.AppStoreGitOpsResponse if installedAppDeploymentAction.PerformGitOps { - gitOpsResponse, err = impl.appStoreDeploymentCommonService.GitOpsOperations(manifest, installAppVersionRequest) + manifest, err := impl.appStoreDeploymentArgoCdService.GenerateManifest(installAppVersionRequest) + if err != nil { + impl.logger.Errorw("error in performing manifest and git operations", "err", err) + return nil, err + } + gitOpsResponse, err = impl.appStoreDeploymentArgoCdService.GitOpsOperations(manifest, installAppVersionRequest) if err != nil { impl.logger.Errorw("error in doing gitops operation", "err", err) if util.IsAcdApp(installAppVersionRequest.DeploymentAppType) { @@ -455,9 +460,7 @@ func (impl AppStoreDeploymentServiceImpl) DeleteInstalledApp(ctx context.Context } if installAppVersionRequest.AcdPartialDelete == true { - if util2.IsBaseStack() || util2.IsHelmApp(app.AppOfferingMode) || util.IsHelmApp(model.DeploymentAppType) { - err = impl.appStoreDeploymentHelmService.DeleteDeploymentApp(ctx, app.AppName, environment.Name, installAppVersionRequest) - } else { + if !util2.IsBaseStack() && !util2.IsHelmApp(app.AppOfferingMode) && !util.IsHelmApp(model.DeploymentAppType) { if !installAppVersionRequest.InstalledAppDeleteResponse.ClusterReachable { impl.logger.Errorw("cluster connection error", "err", environment.Cluster.ErrorInConnecting) if !installAppVersionRequest.NonCascadeDelete { @@ -554,7 +557,7 @@ func (impl AppStoreDeploymentServiceImpl) DeleteInstalledApp(ctx context.Context } func (impl AppStoreDeploymentServiceImpl) LinkHelmApplicationToChartStore(ctx context.Context, request *openapi.UpdateReleaseWithChartLinkingRequest, - appIdentifier *client.AppIdentifier, userId int32) (*openapi.UpdateReleaseResponse, bool, error) { + appIdentifier *service.AppIdentifier, userId int32) (*openapi.UpdateReleaseResponse, bool, error) { impl.logger.Infow("Linking helm application to chart store", "appId", request.GetAppId()) @@ -732,20 +735,20 @@ func (impl AppStoreDeploymentServiceImpl) linkHelmApplicationToChartStore(instal // STEP-2 update APP with chart info chartRepoInfo := appStoreAppVersion.AppStore.ChartRepo - updateReleaseRequest := &client.UpdateApplicationWithChartInfoRequestDto{ - InstallReleaseRequest: &client.InstallReleaseRequest{ + updateReleaseRequest := &bean3.UpdateApplicationWithChartInfoRequestDto{ + InstallReleaseRequest: &bean4.InstallReleaseRequest{ ValuesYaml: installAppVersionRequest.ValuesOverrideYaml, ChartName: appStoreAppVersion.Name, ChartVersion: appStoreAppVersion.Version, - ReleaseIdentifier: &client.ReleaseIdentifier{ + ReleaseIdentifier: &bean4.ReleaseIdentifier{ ReleaseNamespace: installAppVersionRequest.Namespace, ReleaseName: installAppVersionRequest.AppName, }, }, - SourceAppType: client.SOURCE_HELM_APP, + SourceAppType: bean3.SOURCE_HELM_APP, } if chartRepoInfo != nil { - updateReleaseRequest.ChartRepository = &client.ChartRepository{ + updateReleaseRequest.ChartRepository = &bean4.ChartRepository{ Name: chartRepoInfo.Name, Url: chartRepoInfo.Url, Username: chartRepoInfo.UserName, @@ -829,8 +832,8 @@ func (impl AppStoreDeploymentServiceImpl) updateInstalledAppVersionHistoryWithSy } return nil } -func (impl AppStoreDeploymentServiceImpl) GetDeploymentHistory(ctx context.Context, installedApp *appStoreBean.InstallAppVersionDTO) (*client.DeploymentHistoryAndInstalledAppInfo, error) { - result := &client.DeploymentHistoryAndInstalledAppInfo{} +func (impl AppStoreDeploymentServiceImpl) GetDeploymentHistory(ctx context.Context, installedApp *appStoreBean.InstallAppVersionDTO) (*bean3.DeploymentHistoryAndInstalledAppInfo, error) { + result := &bean3.DeploymentHistoryAndInstalledAppInfo{} var err error if util2.IsHelmApp(installedApp.AppOfferingMode) { deploymentHistory, err := impl.appStoreDeploymentHelmService.GetDeploymentHistory(ctx, installedApp) @@ -849,7 +852,7 @@ func (impl AppStoreDeploymentServiceImpl) GetDeploymentHistory(ctx context.Conte } if installedApp.InstalledAppId > 0 { - result.InstalledAppInfo = &client.InstalledAppInfo{ + result.InstalledAppInfo = &bean3.InstalledAppInfo{ AppId: installedApp.AppId, EnvironmentName: installedApp.EnvironmentName, AppOfferingMode: installedApp.AppOfferingMode, @@ -1012,6 +1015,8 @@ func (impl *AppStoreDeploymentServiceImpl) MarkInstalledAppVersionModelInActive( } func (impl *AppStoreDeploymentServiceImpl) CreateInstalledAppVersion(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, tx *pg.Tx) (*repository.InstalledAppVersions, error) { + // TODO fix me next + // TODO refactoring: move this to adapter installedAppVersion := &repository.InstalledAppVersions{ InstalledAppId: installAppVersionRequest.InstalledAppId, AppStoreApplicationVersionId: installAppVersionRequest.AppStoreVersion, @@ -1040,7 +1045,7 @@ func (impl *AppStoreDeploymentServiceImpl) CheckIfMonoRepoMigrationRequired(inst gitOpsRepoName := installedApp.GitOpsRepoName if len(gitOpsRepoName) == 0 { if util.IsAcdApp(installAppVersionRequest.DeploymentAppType) { - gitOpsRepoName, err = impl.appStoreDeploymentArgoCdService.GetGitOpsRepoName(installAppVersionRequest.AppName, installAppVersionRequest.EnvironmentName) + gitOpsRepoName, err = impl.appStoreDeploymentArgoCdService.GetAcdAppGitOpsRepoName(installAppVersionRequest.AppName, installAppVersionRequest.EnvironmentName) if err != nil { return false } @@ -1116,6 +1121,8 @@ func (impl *AppStoreDeploymentServiceImpl) UpdateInstalledApp(ctx context.Contex } installedAppVersion.Id = installedAppVersion.Id } else { + // TODO fix me next + // TODO refactoring: move this to adapter installedAppVersion.ValuesYaml = installAppVersionRequest.ValuesOverrideYaml installedAppVersion.UpdatedOn = time.Now() installedAppVersion.UpdatedBy = installAppVersionRequest.UserId @@ -1139,6 +1146,8 @@ func (impl *AppStoreDeploymentServiceImpl) UpdateInstalledApp(ctx context.Contex installAppVersionRequest.Environment = &installedApp.Environment installAppVersionHistoryStatus := pipelineConfig.WorkflowInProgress + // TODO fix me next + // TODO refactoring: move this to adapter installedAppVersionHistory := &repository.InstalledAppVersionHistory{ InstalledAppVersionId: installedAppVersion.Id, ValuesYamlRaw: installAppVersionRequest.ValuesOverrideYaml, @@ -1159,26 +1168,25 @@ func (impl *AppStoreDeploymentServiceImpl) UpdateInstalledApp(ctx context.Contex installAppVersionRequest.InstalledAppVersionHistoryId = installedAppVersionHistory.Id _ = impl.appStoreDeploymentArgoCdService.SaveTimelineForACDHelmApps(installAppVersionRequest, pipelineConfig.TIMELINE_STATUS_DEPLOYMENT_INITIATED, "Deployment initiated successfully.", time.Now(), tx) - manifest, err := impl.appStoreDeploymentCommonService.GenerateManifest(installAppVersionRequest) - if err != nil { - impl.logger.Errorw("error in generating manifest for helm apps", "err", err) - _ = impl.UpdateInstalledAppVersionHistoryStatus(installAppVersionRequest, pipelineConfig.WorkflowFailed) - return nil, err - } - if installAppVersionRequest.DeploymentAppType == util.PIPELINE_DEPLOYMENT_TYPE_MANIFEST_DOWNLOAD { + if util.IsManifestDownload(installAppVersionRequest.DeploymentAppType) { _ = impl.appStoreDeploymentArgoCdService.SaveTimelineForACDHelmApps(installAppVersionRequest, pipelineConfig.TIMELINE_DESCRIPTION_MANIFEST_GENERATED, "Manifest generated successfully.", time.Now(), tx) } // gitOps operation monoRepoMigrationRequired := false - gitOpsResponse := &appStoreDeploymentCommon.AppStoreGitOpsResponse{} + gitOpsResponse := &bean2.AppStoreGitOpsResponse{} if installedAppDeploymentAction.PerformGitOps { - + manifest, err := impl.appStoreDeploymentArgoCdService.GenerateManifest(installAppVersionRequest) + if err != nil { + impl.logger.Errorw("error in generating manifest for helm apps", "err", err) + _ = impl.UpdateInstalledAppVersionHistoryStatus(installAppVersionRequest, pipelineConfig.WorkflowFailed) + return nil, err + } // required if gitOps repo name is changed, gitOps repo name will change if env variable which we use as suffix changes gitOpsRepoName := installedApp.GitOpsRepoName if len(gitOpsRepoName) == 0 { if util.IsAcdApp(installAppVersionRequest.DeploymentAppType) { - gitOpsRepoName, err = impl.appStoreDeploymentArgoCdService.GetGitOpsRepoName(installAppVersionRequest.AppName, installAppVersionRequest.EnvironmentName) + gitOpsRepoName, err = impl.appStoreDeploymentArgoCdService.GetAcdAppGitOpsRepoName(installAppVersionRequest.AppName, installAppVersionRequest.EnvironmentName) if err != nil { return nil, err } @@ -1201,10 +1209,10 @@ func (impl *AppStoreDeploymentServiceImpl) UpdateInstalledApp(ctx context.Contex var createRepoErr, requirementsCommitErr, valuesCommitErr error var gitHash string - + // TODO refactoring: move this logic to AppStoreDeploymentGitService.go if monoRepoMigrationRequired { // create new git repo if repo name changed - gitOpsResponse, createRepoErr = impl.appStoreDeploymentCommonService.GitOpsOperations(manifest, installAppVersionRequest) + gitOpsResponse, createRepoErr = impl.appStoreDeploymentArgoCdService.GitOpsOperations(manifest, installAppVersionRequest) gitHash = gitOpsResponse.GitHash } else if isChartChanged || isVersionChanged { @@ -1219,13 +1227,13 @@ func (impl *AppStoreDeploymentServiceImpl) UpdateInstalledApp(ctx context.Contex if valuesCommitErr != nil || requirementsCommitErr != nil { - noTargetFoundForValues, _ := impl.appStoreDeploymentCommonService.ParseGitRepoErrorResponse(valuesCommitErr) - noTargetFoundForRequirements, _ := impl.appStoreDeploymentCommonService.ParseGitRepoErrorResponse(requirementsCommitErr) + noTargetFoundForValues, _ := impl.appStoreDeploymentArgoCdService.ParseGitRepoErrorResponse(valuesCommitErr) + noTargetFoundForRequirements, _ := impl.appStoreDeploymentArgoCdService.ParseGitRepoErrorResponse(requirementsCommitErr) if noTargetFoundForRequirements || noTargetFoundForValues { //create repo again and try again - auto fix monoRepoMigrationRequired = true // since repo is created again, will use this flag to check if ACD patch operation required - gitOpsResponse, createRepoErr = impl.appStoreDeploymentCommonService.GitOpsOperations(manifest, installAppVersionRequest) + gitOpsResponse, createRepoErr = impl.appStoreDeploymentArgoCdService.GitOpsOperations(manifest, installAppVersionRequest) gitHash = gitOpsResponse.GitHash } @@ -1234,6 +1242,7 @@ func (impl *AppStoreDeploymentServiceImpl) UpdateInstalledApp(ctx context.Contex if createRepoErr != nil || requirementsCommitErr != nil || valuesCommitErr != nil { impl.logger.Errorw("error in doing gitops operation", "err", err) _ = impl.appStoreDeploymentArgoCdService.SaveTimelineForACDHelmApps(installAppVersionRequest, pipelineConfig.TIMELINE_STATUS_GIT_COMMIT_FAILED, fmt.Sprintf("Git commit failed - %v", err), time.Now(), tx) + // TODO refactoring: return proper err object return nil, err } diff --git a/pkg/appStore/deployment/service/AppStoreDeploymentService_test.go b/pkg/appStore/deployment/service/AppStoreDeploymentService_test.go index e0589afb1b..81eae67783 100644 --- a/pkg/appStore/deployment/service/AppStoreDeploymentService_test.go +++ b/pkg/appStore/deployment/service/AppStoreDeploymentService_test.go @@ -173,9 +173,9 @@ func initAppStoreDeploymentService(t *testing.T, internalUse bool) *AppStoreDepl nil, InstalledAppVersionHistoryRepository, gitOpsRepository, - nil, &DeploymentServiceTypeConfig{IsInternalUse: internalUse}, nil, + nil, nil) return AppStoreDeploymentServiceImpl diff --git a/pkg/appStore/deployment/service/InstalledAppService.go b/pkg/appStore/deployment/service/InstalledAppService.go index d7334501f6..8f5ad9002f 100644 --- a/pkg/appStore/deployment/service/InstalledAppService.go +++ b/pkg/appStore/deployment/service/InstalledAppService.go @@ -20,9 +20,12 @@ package service import ( "bytes" "context" + "github.com/devtron-labs/devtron/api/helm-app/gRPC" + client "github.com/devtron-labs/devtron/api/helm-app/service" + appStoreDeploymentTool "github.com/devtron-labs/devtron/pkg/appStore/deployment/tool" commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" - "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote" + util2 "github.com/devtron-labs/devtron/pkg/util" /* #nosec */ "crypto/sha1" @@ -38,7 +41,6 @@ import ( "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" "github.com/devtron-labs/common-lib/pubsub-lib/model" util4 "github.com/devtron-labs/common-lib/utils/k8s" - client "github.com/devtron-labs/devtron/api/helm-app" openapi "github.com/devtron-labs/devtron/api/helm-app/openapiClient" "github.com/devtron-labs/devtron/client/argocdServer" "github.com/devtron-labs/devtron/internal/middleware" @@ -49,8 +51,6 @@ import ( appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" "github.com/devtron-labs/devtron/pkg/appStore/chartGroup" repository6 "github.com/devtron-labs/devtron/pkg/appStore/chartGroup/repository" - appStoreDeploymentCommon "github.com/devtron-labs/devtron/pkg/appStore/deployment/common" - appStoreDeploymentFullMode "github.com/devtron-labs/devtron/pkg/appStore/deployment/fullMode" repository2 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" appStoreDiscoverRepository "github.com/devtron-labs/devtron/pkg/appStore/discover/repository" "github.com/devtron-labs/devtron/pkg/appStore/values/service" @@ -60,7 +60,6 @@ import ( application3 "github.com/devtron-labs/devtron/pkg/k8s/application" "github.com/devtron-labs/devtron/pkg/sql" repository4 "github.com/devtron-labs/devtron/pkg/team" - util2 "github.com/devtron-labs/devtron/pkg/util" util3 "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/argo" @@ -69,7 +68,6 @@ import ( pubsub "github.com/devtron-labs/common-lib/pubsub-lib" bean2 "github.com/devtron-labs/devtron/api/bean" application2 "github.com/devtron-labs/devtron/client/argocdServer/application" - repository3 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/bean" cluster2 "github.com/devtron-labs/devtron/pkg/cluster" @@ -108,23 +106,20 @@ type InstalledAppServiceImpl struct { chartGroupDeploymentRepository repository6.ChartGroupDeploymentRepository envService cluster2.EnvironmentService aCDAuthConfig *util2.ACDAuthConfig - gitOpsRepository repository3.GitOpsConfigRepository userService user.UserService appStoreDeploymentService AppStoreDeploymentService - appStoreDeploymentFullModeService appStoreDeploymentFullMode.AppStoreDeploymentFullModeService installedAppRepositoryHistory repository2.InstalledAppVersionHistoryRepository argoUserService argo.ArgoUserService - helmAppClient client.HelmAppClient + helmAppClient gRPC.HelmAppClient helmAppService client.HelmAppService appStatusService appStatus.AppStatusService K8sUtil *util4.K8sUtil pipelineStatusTimelineService status.PipelineStatusTimelineService - appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService k8sCommonService k8s.K8sCommonService k8sApplicationService application3.K8sApplicationService acdConfig *argocdServer.ACDConfig - gitOpsConfigReadService config.GitOpsConfigReadService gitOpsRemoteOperationService remote.GitOpsRemoteOperationService + appStoreDeploymentArgoCdService appStoreDeploymentTool.AppStoreDeploymentArgoCdService } func NewInstalledAppServiceImpl(logger *zap.SugaredLogger, @@ -137,17 +132,16 @@ func NewInstalledAppServiceImpl(logger *zap.SugaredLogger, pubsubClient *pubsub.PubSubClientServiceImpl, chartGroupDeploymentRepository repository6.ChartGroupDeploymentRepository, envService cluster2.EnvironmentService, - aCDAuthConfig *util2.ACDAuthConfig, gitOpsRepository repository3.GitOpsConfigRepository, userService user.UserService, - appStoreDeploymentFullModeService appStoreDeploymentFullMode.AppStoreDeploymentFullModeService, + userService user.UserService, aCDAuthConfig *util2.ACDAuthConfig, appStoreDeploymentService AppStoreDeploymentService, installedAppRepositoryHistory repository2.InstalledAppVersionHistoryRepository, - argoUserService argo.ArgoUserService, helmAppClient client.HelmAppClient, helmAppService client.HelmAppService, + argoUserService argo.ArgoUserService, helmAppClient gRPC.HelmAppClient, helmAppService client.HelmAppService, appStatusService appStatus.AppStatusService, K8sUtil *util4.K8sUtil, pipelineStatusTimelineService status.PipelineStatusTimelineService, - appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService, k8sCommonService k8s.K8sCommonService, k8sApplicationService application3.K8sApplicationService, - acdConfig *argocdServer.ACDConfig, gitOpsConfigReadService config.GitOpsConfigReadService, - gitOpsRemoteOperationService remote.GitOpsRemoteOperationService) (*InstalledAppServiceImpl, error) { + acdConfig *argocdServer.ACDConfig, + gitOpsRemoteOperationService remote.GitOpsRemoteOperationService, + appStoreDeploymentArgoCdService appStoreDeploymentTool.AppStoreDeploymentArgoCdService) (*InstalledAppServiceImpl, error) { impl := &InstalledAppServiceImpl{ logger: logger, installedAppRepository: installedAppRepository, @@ -161,10 +155,8 @@ func NewInstalledAppServiceImpl(logger *zap.SugaredLogger, chartGroupDeploymentRepository: chartGroupDeploymentRepository, envService: envService, aCDAuthConfig: aCDAuthConfig, - gitOpsRepository: gitOpsRepository, userService: userService, appStoreDeploymentService: appStoreDeploymentService, - appStoreDeploymentFullModeService: appStoreDeploymentFullModeService, installedAppRepositoryHistory: installedAppRepositoryHistory, argoUserService: argoUserService, helmAppClient: helmAppClient, @@ -172,19 +164,59 @@ func NewInstalledAppServiceImpl(logger *zap.SugaredLogger, appStatusService: appStatusService, K8sUtil: K8sUtil, pipelineStatusTimelineService: pipelineStatusTimelineService, - appStoreDeploymentCommonService: appStoreDeploymentCommonService, k8sCommonService: k8sCommonService, k8sApplicationService: k8sApplicationService, acdConfig: acdConfig, - gitOpsConfigReadService: gitOpsConfigReadService, gitOpsRemoteOperationService: gitOpsRemoteOperationService, + appStoreDeploymentArgoCdService: appStoreDeploymentArgoCdService, } err := impl.subscribe() if err != nil { return nil, err } + err = impl.subscribeHelmInstallStatus() + if err != nil { + return nil, err + } return impl, nil } +func (impl InstalledAppServiceImpl) subscribeHelmInstallStatus() error { + + callback := func(msg *model.PubSubMsg) { + + impl.logger.Debug("received helm install status event - HELM_INSTALL_STATUS", "data", msg.Data) + helmInstallNatsMessage := &appStoreBean.HelmReleaseStatusConfig{} + err := json.Unmarshal([]byte(msg.Data), helmInstallNatsMessage) + if err != nil { + impl.logger.Errorw("error in unmarshalling helm install status nats message", "err", err) + return + } + + installedAppVersionHistory, err := impl.installedAppRepositoryHistory.GetInstalledAppVersionHistory(helmInstallNatsMessage.InstallAppVersionHistoryId) + if err != nil { + impl.logger.Errorw("error in fetching installed app by installed app id in subscribe helm status callback", "err", err) + return + } + if helmInstallNatsMessage.ErrorInInstallation { + installedAppVersionHistory.Status = pipelineConfig.WorkflowFailed + } else { + installedAppVersionHistory.Status = pipelineConfig.WorkflowSucceeded + } + installedAppVersionHistory.HelmReleaseStatusConfig = msg.Data + _, err = impl.installedAppRepositoryHistory.UpdateInstalledAppVersionHistory(installedAppVersionHistory, nil) + if err != nil { + impl.logger.Errorw("error in updating helm release status data in installedAppVersionHistoryRepository", "err", err) + return + } + } + + err := impl.pubsubClient.Subscribe(pubsub.HELM_CHART_INSTALL_STATUS_TOPIC, callback) + if err != nil { + impl.logger.Error(err) + return err + } + return nil +} func (impl InstalledAppServiceImpl) GetAll(filter *appStoreBean.AppStoreFilter) (openapi.AppList, error) { applicationType := "DEVTRON-CHART-STORE" @@ -363,7 +395,7 @@ func (impl InstalledAppServiceImpl) performDeployStageOnAcd(installedAppVersion installedAppVersion.Status == appStoreBean.GIT_ERROR { //step 2 git operation pull push //TODO: save git Timeline here - appStoreGitOpsResponse, err := impl.appStoreDeploymentCommonService.GenerateManifestAndPerformGitOperations(installedAppVersion) + appStoreGitOpsResponse, err := impl.appStoreDeploymentArgoCdService.GenerateManifestAndPerformGitOperations(installedAppVersion) if err != nil { impl.logger.Errorw(" error", "err", err) _, err = impl.appStoreDeploymentService.AppStoreDeployOperationStatusUpdate(installedAppVersion.InstalledAppId, appStoreBean.GIT_ERROR) @@ -465,7 +497,7 @@ func (impl InstalledAppServiceImpl) performDeployStageOnAcd(installedAppVersion installedAppVersion.Status == appStoreBean.GIT_SUCCESS || installedAppVersion.Status == appStoreBean.ACD_ERROR { //step 3 acd operation register, sync - _, err := impl.appStoreDeploymentFullModeService.AppStoreDeployOperationACD(installedAppVersion, chartGitAttr, ctx, nil) + _, err := impl.appStoreDeploymentArgoCdService.InstallApp(installedAppVersion, chartGitAttr, ctx, nil) if err != nil { impl.logger.Errorw("error", "chartGitAttr", chartGitAttr, "err", err) _, err = impl.appStoreDeploymentService.AppStoreDeployOperationStatusUpdate(installedAppVersion.InstalledAppId, appStoreBean.ACD_ERROR) @@ -963,9 +995,9 @@ func (impl InstalledAppServiceImpl) GetInstalledAppVersionHistoryValues(installe return values, err } -func (impl InstalledAppServiceImpl) getReleaseStatusFromHelmReleaseInstallStatus(helmReleaseInstallStatus string, status string) *client.ReleaseStatus { +func (impl InstalledAppServiceImpl) getReleaseStatusFromHelmReleaseInstallStatus(helmReleaseInstallStatus string, status string) *gRPC.ReleaseStatus { //release status is sent in resource tree call and is shown on UI as helm config apply status - releaseStatus := &client.ReleaseStatus{} + releaseStatus := &gRPC.ReleaseStatus{} if len(helmReleaseInstallStatus) > 0 { helmInstallStatus := &appStoreBean.HelmReleaseStatusConfig{} err := json.Unmarshal([]byte(helmReleaseInstallStatus), helmInstallStatus) @@ -995,6 +1027,7 @@ func (impl InstalledAppServiceImpl) getReleaseStatusFromHelmReleaseInstallStatus return releaseStatus } +// TODO fix me next func (impl InstalledAppServiceImpl) MarkGitOpsInstalledAppsDeletedIfArgoAppIsDeleted(installedAppId int, envId int) error { apiError := &util.ApiError{} installedApp, err := impl.installedAppRepository.GetGitOpsInstalledAppsWhereArgoAppDeletedIsTrue(installedAppId, envId) @@ -1004,6 +1037,7 @@ func (impl InstalledAppServiceImpl) MarkGitOpsInstalledAppsDeletedIfArgoAppIsDel apiError.InternalMessage = "error in fetching partially deleted argoCd apps from installed app repo" return apiError } + // TODO refactoring: move the below logic to AppStoreDeploymentArgoCdService.go acdToken, err := impl.argoUserService.GetLatestDevtronArgoCdUserToken() if err != nil { impl.logger.Errorw("error in getting acd token", "err", err) diff --git a/pkg/appStore/deployment/service/Notes.go b/pkg/appStore/deployment/service/Notes.go index 33595afc25..cd008184eb 100644 --- a/pkg/appStore/deployment/service/Notes.go +++ b/pkg/appStore/deployment/service/Notes.go @@ -3,7 +3,7 @@ package service import ( "context" "fmt" - "github.com/devtron-labs/devtron/api/helm-app" + "github.com/devtron-labs/devtron/api/helm-app/gRPC" "github.com/devtron-labs/devtron/internal/util" "github.com/go-pg/pg" "regexp" @@ -78,18 +78,18 @@ func (impl *InstalledAppServiceImpl) findNotesForArgoApplication(installedAppId, return notes, appName, err } - installReleaseRequest := &client.InstallReleaseRequest{ + installReleaseRequest := &gRPC.InstallReleaseRequest{ ChartName: appStoreAppVersion.Name, ChartVersion: appStoreAppVersion.Version, ValuesYaml: installedAppVerison.ValuesYaml, K8SVersion: k8sServerVersion.String(), - ChartRepository: &client.ChartRepository{ + ChartRepository: &gRPC.ChartRepository{ Name: appStoreAppVersion.AppStore.ChartRepo.Name, Url: appStoreAppVersion.AppStore.ChartRepo.Url, Username: appStoreAppVersion.AppStore.ChartRepo.UserName, Password: appStoreAppVersion.AppStore.ChartRepo.Password, }, - ReleaseIdentifier: &client.ReleaseIdentifier{ + ReleaseIdentifier: &gRPC.ReleaseIdentifier{ ReleaseNamespace: installedAppVerison.InstalledApp.Environment.Namespace, ReleaseName: installedAppVerison.InstalledApp.App.AppName, }, diff --git a/pkg/appStore/deployment/service/ResourceTree.go b/pkg/appStore/deployment/service/ResourceTree.go index e826bacb55..3d0e36b617 100644 --- a/pkg/appStore/deployment/service/ResourceTree.go +++ b/pkg/appStore/deployment/service/ResourceTree.go @@ -8,7 +8,7 @@ import ( "github.com/devtron-labs/common-lib/utils/k8s/commonBean" "github.com/devtron-labs/common-lib/utils/k8sObjectsUtil" "github.com/devtron-labs/devtron/api/bean" - "github.com/devtron-labs/devtron/api/helm-app" + bean2 "github.com/devtron-labs/devtron/api/helm-app/gRPC" "github.com/devtron-labs/devtron/internal/constants" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/appStatus" @@ -35,7 +35,7 @@ func (impl InstalledAppServiceImpl) FetchResourceTree(rctx context.Context, cn h if err != nil { impl.logger.Errorw("error in fetching cluster detail", "err", err) } - req := &client.AppDetailRequest{ + req := &bean2.AppDetailRequest{ ClusterConfig: config, Namespace: installedApp.Environment.Namespace, ReleaseName: installedApp.App.AppName, diff --git a/pkg/appStore/deployment/tool/AppStoreDeploymentArgoCdService.go b/pkg/appStore/deployment/tool/AppStoreDeploymentArgoCdService.go index 6b4361ae7d..839d6e78ff 100644 --- a/pkg/appStore/deployment/tool/AppStoreDeploymentArgoCdService.go +++ b/pkg/appStore/deployment/tool/AppStoreDeploymentArgoCdService.go @@ -5,14 +5,19 @@ import ( "encoding/json" "errors" "fmt" + "github.com/devtron-labs/devtron/api/helm-app/gRPC" + client "github.com/devtron-labs/devtron/api/helm-app/service" + "github.com/devtron-labs/devtron/pkg/appStore/deployment/tool/bean" + repository5 "github.com/devtron-labs/devtron/pkg/cluster/repository" commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote" + util2 "github.com/devtron-labs/devtron/pkg/util" "strings" "time" "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" - client "github.com/devtron-labs/devtron/api/helm-app" openapi "github.com/devtron-labs/devtron/api/helm-app/openapiClient" openapi2 "github.com/devtron-labs/devtron/api/openapi/openapiClient" "github.com/devtron-labs/devtron/client/argocdServer" @@ -24,7 +29,6 @@ import ( appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" repository2 "github.com/devtron-labs/devtron/pkg/appStore/chartGroup/repository" appStoreDeploymentCommon "github.com/devtron-labs/devtron/pkg/appStore/deployment/common" - appStoreDeploymentFullMode "github.com/devtron-labs/devtron/pkg/appStore/deployment/fullMode" "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" appStoreDiscoverRepository "github.com/devtron-labs/devtron/pkg/appStore/discover/repository" "github.com/devtron-labs/devtron/pkg/auth/user" @@ -40,23 +44,47 @@ import ( // creating duplicates because cannot use type AppStoreDeploymentArgoCdService interface { - //InstallApp(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, ctx context.Context) (*appStoreBean.InstallAppVersionDTO, error) + // ArgoCd Services --------------------------------- + + // InstallApp will register git repo in Argo, create and sync the Argo App and finally update deployment status InstallApp(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, chartGitAttr *commonBean.ChartGitAttribute, ctx context.Context, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) + // DeleteInstalledApp will delete entry from appStatus.AppStatusDto table and from repository.ChartGroupDeployment table (if exists) DeleteInstalledApp(ctx context.Context, appName string, environmentName string, installAppVersionRequest *appStoreBean.InstallAppVersionDTO, installedApps *repository.InstalledApps, dbTransaction *pg.Tx) error + // RollbackRelease will rollback to a previous deployment for the given installedAppVersionHistoryId; returns - valuesYamlStr, success, error RollbackRelease(ctx context.Context, installedApp *appStoreBean.InstallAppVersionDTO, deploymentVersion int32, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, bool, error) - GetDeploymentHistory(ctx context.Context, installedApp *appStoreBean.InstallAppVersionDTO) (*client.HelmAppDeploymentHistory, error) + // GetDeploymentHistory will return gRPC.HelmAppDeploymentHistory for the given installedAppDto.InstalledAppId + GetDeploymentHistory(ctx context.Context, installedApp *appStoreBean.InstallAppVersionDTO) (*gRPC.HelmAppDeploymentHistory, error) + // GetDeploymentHistoryInfo will return openapi.HelmAppDeploymentManifestDetail for the given appStoreBean.InstallAppVersionDTO GetDeploymentHistoryInfo(ctx context.Context, installedApp *appStoreBean.InstallAppVersionDTO, version int32) (*openapi.HelmAppDeploymentManifestDetail, error) - GetGitOpsRepoName(appName string, environmentName string) (string, error) + // GetAcdAppGitOpsRepoName will return the Git Repo used in the ACD app object + GetAcdAppGitOpsRepoName(appName string, environmentName string) (string, error) + // DeleteDeploymentApp will delete the app object from ACD DeleteDeploymentApp(ctx context.Context, appName string, environmentName string, installAppVersionRequest *appStoreBean.InstallAppVersionDTO) error - UpdateInstalledAppAndPipelineStatusForFailedDeploymentStatus(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, triggeredAt time.Time, err error) error SaveTimelineForACDHelmApps(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, status string, statusDetail string, statusTime time.Time, tx *pg.Tx) error UpdateAndSyncACDApps(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, ChartGitAttribute *commonBean.ChartGitAttribute, isMonoRepoMigrationRequired bool, ctx context.Context, tx *pg.Tx) error + + // Deployment Status Services ---------------------- + + // UpdateInstalledAppAndPipelineStatusForFailedDeploymentStatus updates failed status in pipelineConfig.PipelineStatusTimeline table + UpdateInstalledAppAndPipelineStatusForFailedDeploymentStatus(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, triggeredAt time.Time, err error) error + + // Git Operation Services -------------------------- + + // ParseGitRepoErrorResponse will return noTargetFound (if git API returns 404 status) + ParseGitRepoErrorResponse(err error) (bool, error) + // GitOpsOperations performs git operations specific to helm app deployments + GitOpsOperations(manifestResponse *bean.AppStoreManifestResponse, installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*bean.AppStoreGitOpsResponse, error) + // GenerateManifest returns bean.AppStoreManifestResponse required in GitOps + GenerateManifest(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (manifestResponse *bean.AppStoreManifestResponse, err error) + // GenerateManifestAndPerformGitOperations is a wrapper function for both GenerateManifest and GitOpsOperations + GenerateManifestAndPerformGitOperations(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*bean.AppStoreGitOpsResponse, error) } type AppStoreDeploymentArgoCdServiceImpl struct { Logger *zap.SugaredLogger - appStoreDeploymentFullModeService appStoreDeploymentFullMode.AppStoreDeploymentFullModeService acdClient application2.ServiceClient + argoK8sClient argocdServer.ArgoK8sClient + aCDAuthConfig *util2.ACDAuthConfig chartGroupDeploymentRepository repository2.ChartGroupDeploymentRepository installedAppRepository repository.InstalledAppRepository installedAppRepositoryHistory repository.InstalledAppVersionHistoryRepository @@ -71,22 +99,36 @@ type AppStoreDeploymentArgoCdServiceImpl struct { argoClientWrapperService argocdServer.ArgoClientWrapperService acdConfig *argocdServer.ACDConfig gitOpsRemoteOperationService remote.GitOpsRemoteOperationService + gitOpsConfigReadService config.GitOpsConfigReadService + environmentRepository repository5.EnvironmentRepository } -func NewAppStoreDeploymentArgoCdServiceImpl(logger *zap.SugaredLogger, appStoreDeploymentFullModeService appStoreDeploymentFullMode.AppStoreDeploymentFullModeService, - acdClient application2.ServiceClient, chartGroupDeploymentRepository repository2.ChartGroupDeploymentRepository, - installedAppRepository repository.InstalledAppRepository, installedAppRepositoryHistory repository.InstalledAppVersionHistoryRepository, - argoUserService argo.ArgoUserService, appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService, - helmAppService client.HelmAppService, appStatusService appStatus.AppStatusService, - pipelineStatusTimelineService status.PipelineStatusTimelineService, userService user.UserService, +func NewAppStoreDeploymentArgoCdServiceImpl( + logger *zap.SugaredLogger, + acdClient application2.ServiceClient, + argoK8sClient argocdServer.ArgoK8sClient, + aCDAuthConfig *util2.ACDAuthConfig, + chartGroupDeploymentRepository repository2.ChartGroupDeploymentRepository, + installedAppRepository repository.InstalledAppRepository, + installedAppRepositoryHistory repository.InstalledAppVersionHistoryRepository, + argoUserService argo.ArgoUserService, + appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService, + helmAppService client.HelmAppService, + appStatusService appStatus.AppStatusService, + pipelineStatusTimelineService status.PipelineStatusTimelineService, + userService user.UserService, pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository, appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository, - argoClientWrapperService argocdServer.ArgoClientWrapperService, acdConfig *argocdServer.ACDConfig, - gitOpsRemoteOperationService remote.GitOpsRemoteOperationService) *AppStoreDeploymentArgoCdServiceImpl { + argoClientWrapperService argocdServer.ArgoClientWrapperService, + acdConfig *argocdServer.ACDConfig, + gitOpsRemoteOperationService remote.GitOpsRemoteOperationService, + gitOpsConfigReadService config.GitOpsConfigReadService, + environmentRepository repository5.EnvironmentRepository) *AppStoreDeploymentArgoCdServiceImpl { return &AppStoreDeploymentArgoCdServiceImpl{ Logger: logger, - appStoreDeploymentFullModeService: appStoreDeploymentFullModeService, acdClient: acdClient, + argoK8sClient: argoK8sClient, + aCDAuthConfig: aCDAuthConfig, chartGroupDeploymentRepository: chartGroupDeploymentRepository, installedAppRepository: installedAppRepository, installedAppRepositoryHistory: installedAppRepositoryHistory, @@ -101,6 +143,8 @@ func NewAppStoreDeploymentArgoCdServiceImpl(logger *zap.SugaredLogger, appStoreD argoClientWrapperService: argoClientWrapperService, acdConfig: acdConfig, gitOpsRemoteOperationService: gitOpsRemoteOperationService, + gitOpsConfigReadService: gitOpsConfigReadService, + environmentRepository: environmentRepository, } } @@ -177,31 +221,52 @@ func (impl AppStoreDeploymentArgoCdServiceImpl) SaveTimelineForACDHelmApps(insta } func (impl AppStoreDeploymentArgoCdServiceImpl) InstallApp(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, chartGitAttr *commonBean.ChartGitAttribute, ctx context.Context, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) { + ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) + defer cancel() + //STEP 4: registerInArgo + err := impl.argoClientWrapperService.RegisterGitOpsRepoInArgo(ctx, chartGitAttr.RepoUrl) + if err != nil { + impl.Logger.Errorw("error in argo registry", "err", err) + return nil, err + } + //STEP 5: createInArgo + err = impl.createInArgo(chartGitAttr, *installAppVersionRequest.Environment, installAppVersionRequest.ACDAppName) + if err != nil { + impl.Logger.Errorw("error in create in argo", "err", err) + return nil, err + } + //STEP 6: Force Sync ACD - works like trigger deployment + //impl.SyncACD(installAppVersionRequest.ACDAppName, ctx) - installAppVersionRequest, err := impl.appStoreDeploymentFullModeService.AppStoreDeployOperationACD(installAppVersionRequest, chartGitAttr, ctx, tx) + //STEP 7: normal refresh ACD - update for step 6 to avoid delay + syncTime := time.Now() + err = impl.argoClientWrapperService.SyncArgoCDApplicationIfNeededAndRefresh(ctx, installAppVersionRequest.ACDAppName) if err != nil { - impl.Logger.Errorw(" error", "err", err) - return installAppVersionRequest, err + impl.Logger.Errorw("error in getting the argo application with normal refresh", "err", err) + return nil, err } + if !impl.acdConfig.ArgoCDAutoSyncEnabled { + timeline := &pipelineConfig.PipelineStatusTimeline{ + InstalledAppVersionHistoryId: installAppVersionRequest.InstalledAppVersionHistoryId, + Status: pipelineConfig.TIMELINE_STATUS_ARGOCD_SYNC_COMPLETED, + StatusDetail: "argocd sync completed.", + StatusTime: syncTime, + AuditLog: sql.AuditLog{ + CreatedBy: installAppVersionRequest.UserId, + CreatedOn: time.Now(), + UpdatedBy: installAppVersionRequest.UserId, + UpdatedOn: time.Now(), + }, + } + err = impl.pipelineStatusTimelineService.SaveTimeline(timeline, tx, true) + if err != nil { + impl.Logger.Errorw("error in creating timeline for argocd sync", "err", err, "timeline", timeline) + } + } + return installAppVersionRequest, nil } -//func (impl AppStoreDeploymentArgoCdServiceImpl) InstallApp(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, ctx context.Context) (*appStoreBean.InstallAppVersionDTO, error) { -// //step 2 git operation pull push -// installAppVersionRequest, chartGitAttr, err := impl.appStoreDeploymentFullModeService.AppStoreDeployOperationGIT(installAppVersionRequest) -// if err != nil { -// impl.Logger.Errorw(" error", "err", err) -// return installAppVersionRequest, err -// } -// //step 3 acd operation register, sync -// installAppVersionRequest, err = impl.appStoreDeploymentFullModeService.AppStoreDeployOperationACD(installAppVersionRequest, chartGitAttr, ctx) -// if err != nil { -// impl.Logger.Errorw(" error", "err", err) -// return installAppVersionRequest, err -// } -// return installAppVersionRequest, nil -//} - func (impl AppStoreDeploymentArgoCdServiceImpl) DeleteInstalledApp(ctx context.Context, appName string, environmentName string, installAppVersionRequest *appStoreBean.InstallAppVersionDTO, installedApps *repository.InstalledApps, dbTransaction *pg.Tx) error { err := impl.appStatusService.DeleteWithAppIdEnvId(dbTransaction, installedApps.AppId, installedApps.EnvironmentId) @@ -231,7 +296,6 @@ func (impl AppStoreDeploymentArgoCdServiceImpl) DeleteInstalledApp(ctx context.C return nil } -// returns - valuesYamlStr, success, error func (impl AppStoreDeploymentArgoCdServiceImpl) RollbackRelease(ctx context.Context, installedApp *appStoreBean.InstallAppVersionDTO, installedAppVersionHistoryId int32, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, bool, error) { //request version id for versionHistory, err := impl.installedAppRepositoryHistory.GetInstalledAppVersionHistory(int(installedAppVersionHistoryId)) @@ -294,7 +358,7 @@ func (impl AppStoreDeploymentArgoCdServiceImpl) RollbackRelease(ctx context.Cont } //If current version upgrade/degrade to another, update requirement dependencies if versionHistory.InstalledAppVersionId != activeInstalledAppVersion.Id { - err = impl.appStoreDeploymentFullModeService.UpdateRequirementYaml(installedApp, &installedAppVersion.AppStoreApplicationVersion) + err = impl.updateRequirementYamlInGit(installedApp, &installedAppVersion.AppStoreApplicationVersion) if err != nil { if errors.Is(err, errors.New(pipelineConfig.TIMELINE_STATUS_GIT_COMMIT_FAILED)) { impl.Logger.Errorw("error", "err", err) @@ -312,7 +376,7 @@ func (impl AppStoreDeploymentArgoCdServiceImpl) RollbackRelease(ctx context.Cont } } //Update Values config - installedApp, err = impl.appStoreDeploymentFullModeService.UpdateValuesYaml(installedApp, tx) + installedApp, err = impl.updateValuesYamlInGit(installedApp) if err != nil { impl.Logger.Errorw("error", "err", err) if errors.Is(err, errors.New(pipelineConfig.TIMELINE_STATUS_GIT_COMMIT_FAILED)) { @@ -364,40 +428,9 @@ func (impl AppStoreDeploymentArgoCdServiceImpl) RollbackRelease(ctx context.Cont return installedApp, true, nil } -func (impl AppStoreDeploymentArgoCdServiceImpl) deleteACD(acdAppName string, ctx context.Context, isNonCascade bool) error { - req := new(application.ApplicationDeleteRequest) - req.Name = &acdAppName - cascadeDelete := !isNonCascade - req.Cascade = &cascadeDelete - if ctx == nil { - impl.Logger.Errorw("err in delete ACD for AppStore, ctx is NULL", "acdAppName", acdAppName) - return fmt.Errorf("context is null") - } - if _, err := impl.acdClient.Delete(ctx, req); err != nil { - impl.Logger.Errorw("err in delete ACD for AppStore", "acdAppName", acdAppName, "err", err) - return err - } - return nil -} -func (impl AppStoreDeploymentArgoCdServiceImpl) getSourcesFromManifest(chartYaml string) ([]string, error) { - var b map[string]interface{} - var sources []string - err := json.Unmarshal([]byte(chartYaml), &b) - if err != nil { - impl.Logger.Errorw("error while unmarshal chart yaml", "error", err) - return sources, err - } - if b != nil && b["sources"] != nil { - slice := b["sources"].([]interface{}) - for _, item := range slice { - sources = append(sources, item.(string)) - } - } - return sources, nil -} -func (impl AppStoreDeploymentArgoCdServiceImpl) GetDeploymentHistory(ctx context.Context, installedAppDto *appStoreBean.InstallAppVersionDTO) (*client.HelmAppDeploymentHistory, error) { - result := &client.HelmAppDeploymentHistory{} - var history []*client.HelmAppDeploymentDetail +func (impl AppStoreDeploymentArgoCdServiceImpl) GetDeploymentHistory(ctx context.Context, installedAppDto *appStoreBean.InstallAppVersionDTO) (*gRPC.HelmAppDeploymentHistory, error) { + result := &gRPC.HelmAppDeploymentHistory{} + var history []*gRPC.HelmAppDeploymentDetail //TODO - response setup installedAppVersions, err := impl.installedAppRepository.GetInstalledAppVersionByInstalledAppIdMeta(installedAppDto.InstalledAppId) @@ -430,8 +463,8 @@ func (impl AppStoreDeploymentArgoCdServiceImpl) GetDeploymentHistory(ctx context if user != nil { emailId = user.EmailId } - history = append(history, &client.HelmAppDeploymentDetail{ - ChartMetadata: &client.ChartMetadata{ + history = append(history, &gRPC.HelmAppDeploymentDetail{ + ChartMetadata: &gRPC.ChartMetadata{ ChartName: installedAppVersionModel.AppStoreApplicationVersion.AppStore.Name, ChartVersion: installedAppVersionModel.AppStoreApplicationVersion.Version, Description: installedAppVersionModel.AppStoreApplicationVersion.Description, @@ -451,12 +484,13 @@ func (impl AppStoreDeploymentArgoCdServiceImpl) GetDeploymentHistory(ctx context } if len(history) == 0 { - history = make([]*client.HelmAppDeploymentDetail, 0) + history = make([]*gRPC.HelmAppDeploymentDetail, 0) } result.DeploymentHistory = history return result, err } +// TODO refactoring: use InstalledAppVersionHistoryId from appStoreBean.InstallAppVersionDTO instead of version int32 func (impl AppStoreDeploymentArgoCdServiceImpl) GetDeploymentHistoryInfo(ctx context.Context, installedApp *appStoreBean.InstallAppVersionDTO, version int32) (*openapi.HelmAppDeploymentManifestDetail, error) { values := &openapi.HelmAppDeploymentManifestDetail{} _, span := otel.Tracer("orchestrator").Start(ctx, "installedAppRepositoryHistory.GetInstalledAppVersionHistory") @@ -496,63 +530,27 @@ func (impl AppStoreDeploymentArgoCdServiceImpl) GetDeploymentHistoryInfo(ctx con return values, err } -func (impl AppStoreDeploymentArgoCdServiceImpl) GetGitOpsRepoName(appName string, environmentName string) (string, error) { - return impl.appStoreDeploymentFullModeService.GetGitOpsRepoName(appName, environmentName) -} - -func (impl AppStoreDeploymentArgoCdServiceImpl) patchAcdApp(ctx context.Context, installAppVersionRequest *appStoreBean.InstallAppVersionDTO, chartGitAttr *commonBean.ChartGitAttribute) (*appStoreBean.InstallAppVersionDTO, error) { - ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) - defer cancel() - //registerInArgo - err := impl.argoClientWrapperService.RegisterGitOpsRepoInArgo(ctx, chartGitAttr.RepoUrl) - if err != nil { - impl.Logger.Errorw("error in argo registry", "err", err) - return nil, err - } - // update acd app - patchReq := v1alpha1.Application{Spec: v1alpha1.ApplicationSpec{Source: &v1alpha1.ApplicationSource{Path: chartGitAttr.ChartLocation, RepoURL: chartGitAttr.RepoUrl, TargetRevision: "master"}}} - reqbyte, err := json.Marshal(patchReq) - if err != nil { - impl.Logger.Errorw("error in creating patch", "err", err) - } - reqString := string(reqbyte) - patchType := "merge" - _, err = impl.acdClient.Patch(ctx, &application.ApplicationPatchRequest{Patch: &reqString, Name: &installAppVersionRequest.ACDAppName, PatchType: &patchType}) - if err != nil { - impl.Logger.Errorw("error in creating argo app ", "name", installAppVersionRequest.ACDAppName, "patch", string(reqbyte), "err", err) - return nil, err - } - //impl.appStoreDeploymentFullModeService.SyncACD(installAppVersionRequest.ACDAppName, ctx) - return installAppVersionRequest, nil -} - -func (impl AppStoreDeploymentArgoCdServiceImpl) updateValuesYaml(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) { - - appStoreAppVersion, err := impl.appStoreApplicationVersionRepository.FindById(installAppVersionRequest.AppStoreVersion) - if err != nil { - impl.Logger.Errorw("fetching error", "err", err) - return nil, err - } - valuesString, err := impl.appStoreDeploymentCommonService.GetValuesString(appStoreAppVersion.Name, installAppVersionRequest.ValuesOverrideYaml) +func (impl AppStoreDeploymentArgoCdServiceImpl) GetAcdAppGitOpsRepoName(appName string, environmentName string) (string, error) { + gitOpsRepoName := "" + //this method should only call in case of argo-integration and gitops configured + acdToken, err := impl.argoUserService.GetLatestDevtronArgoCdUserToken() if err != nil { - impl.Logger.Errorw("error in building requirements config for helm app", "err", err) - return nil, err + impl.Logger.Errorw("error in getting acd token", "err", err) + return "", err } - valuesGitConfig, err := impl.appStoreDeploymentCommonService.GetGitCommitConfig(installAppVersionRequest, valuesString, appStoreBean.VALUES_YAML_FILE) + ctx := context.Background() + ctx = context.WithValue(ctx, "token", acdToken) + acdAppName := fmt.Sprintf("%s-%s", appName, environmentName) + acdApplication, err := impl.acdClient.Get(ctx, &application.ApplicationQuery{Name: &acdAppName}) if err != nil { - impl.Logger.Errorw("error in getting git config for helm app", "err", err) - return nil, err + impl.Logger.Errorw("no argo app exists", "acdAppName", acdAppName, "err", err) + return "", err } - gitHash, _, err := impl.gitOpsRemoteOperationService.CommitValues(valuesGitConfig) - if err != nil { - impl.Logger.Errorw("error in git commit", "err", err) - _ = impl.SaveTimelineForACDHelmApps(installAppVersionRequest, pipelineConfig.TIMELINE_STATUS_GIT_COMMIT_FAILED, fmt.Sprintf("Git commit failed - %v", err), time.Now(), tx) - return nil, err + if acdApplication != nil { + gitOpsRepoUrl := acdApplication.Spec.Source.RepoURL + gitOpsRepoName = impl.gitOpsConfigReadService.GetGitOpsRepoNameFromUrl(gitOpsRepoUrl) } - _ = impl.SaveTimelineForACDHelmApps(installAppVersionRequest, pipelineConfig.TIMELINE_STATUS_GIT_COMMIT, "Git commit done successfully.", time.Now(), tx) - //update timeline status for git commit state - installAppVersionRequest.GitHash = gitHash - return installAppVersionRequest, nil + return gitOpsRepoName, nil } func (impl AppStoreDeploymentArgoCdServiceImpl) DeleteDeploymentApp(ctx context.Context, appName string, environmentName string, installAppVersionRequest *appStoreBean.InstallAppVersionDTO) error { @@ -581,3 +579,87 @@ func (impl AppStoreDeploymentArgoCdServiceImpl) DeleteDeploymentApp(ctx context. } return nil } + +func (impl AppStoreDeploymentArgoCdServiceImpl) deleteACD(acdAppName string, ctx context.Context, isNonCascade bool) error { + req := new(application.ApplicationDeleteRequest) + req.Name = &acdAppName + cascadeDelete := !isNonCascade + req.Cascade = &cascadeDelete + if ctx == nil { + impl.Logger.Errorw("err in delete ACD for AppStore, ctx is NULL", "acdAppName", acdAppName) + return fmt.Errorf("context is null") + } + if _, err := impl.acdClient.Delete(ctx, req); err != nil { + impl.Logger.Errorw("err in delete ACD for AppStore", "acdAppName", acdAppName, "err", err) + return err + } + return nil +} + +func (impl AppStoreDeploymentArgoCdServiceImpl) getSourcesFromManifest(chartYaml string) ([]string, error) { + var b map[string]interface{} + var sources []string + err := json.Unmarshal([]byte(chartYaml), &b) + if err != nil { + impl.Logger.Errorw("error while unmarshal chart yaml", "error", err) + return sources, err + } + if b != nil && b["sources"] != nil { + slice := b["sources"].([]interface{}) + for _, item := range slice { + sources = append(sources, item.(string)) + } + } + return sources, nil +} + +func (impl AppStoreDeploymentArgoCdServiceImpl) createInArgo(chartGitAttribute *commonBean.ChartGitAttribute, envModel repository5.Environment, argocdAppName string) error { + appNamespace := envModel.Namespace + if appNamespace == "" { + appNamespace = "default" + } + appreq := &argocdServer.AppTemplate{ + ApplicationName: argocdAppName, + Namespace: impl.aCDAuthConfig.ACDConfigMapNamespace, + TargetNamespace: appNamespace, + TargetServer: envModel.Cluster.ServerUrl, + Project: "default", + ValuesFile: fmt.Sprintf("values.yaml"), + RepoPath: chartGitAttribute.ChartLocation, + RepoUrl: chartGitAttribute.RepoUrl, + AutoSyncEnabled: impl.acdConfig.ArgoCDAutoSyncEnabled, + } + _, err := impl.argoK8sClient.CreateAcdApp(appreq, envModel.Cluster, argocdServer.ARGOCD_APPLICATION_TEMPLATE) + //create + if err != nil { + impl.Logger.Errorw("error in creating argo cd app ", "err", err) + return err + } + return nil +} + +func (impl AppStoreDeploymentArgoCdServiceImpl) patchAcdApp(ctx context.Context, installAppVersionRequest *appStoreBean.InstallAppVersionDTO, chartGitAttr *commonBean.ChartGitAttribute) (*appStoreBean.InstallAppVersionDTO, error) { + ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) + defer cancel() + //registerInArgo + err := impl.argoClientWrapperService.RegisterGitOpsRepoInArgo(ctx, chartGitAttr.RepoUrl) + if err != nil { + impl.Logger.Errorw("error in argo registry", "err", err) + return nil, err + } + // update acd app + patchReq := v1alpha1.Application{Spec: v1alpha1.ApplicationSpec{Source: &v1alpha1.ApplicationSource{Path: chartGitAttr.ChartLocation, RepoURL: chartGitAttr.RepoUrl, TargetRevision: "master"}}} + reqbyte, err := json.Marshal(patchReq) + if err != nil { + impl.Logger.Errorw("error in creating patch", "err", err) + } + reqString := string(reqbyte) + patchType := "merge" + _, err = impl.acdClient.Patch(ctx, &application.ApplicationPatchRequest{Patch: &reqString, Name: &installAppVersionRequest.ACDAppName, PatchType: &patchType}) + if err != nil { + impl.Logger.Errorw("error in creating argo app ", "name", installAppVersionRequest.ACDAppName, "patch", string(reqbyte), "err", err) + return nil, err + } + //impl.appStoreDeploymentFullModeService.SyncACD(installAppVersionRequest.ACDAppName, ctx) + return installAppVersionRequest, nil +} diff --git a/pkg/appStore/deployment/tool/AppStoreDeploymentGitOpsService.go b/pkg/appStore/deployment/tool/AppStoreDeploymentGitOpsService.go new file mode 100644 index 0000000000..164d9a8ed4 --- /dev/null +++ b/pkg/appStore/deployment/tool/AppStoreDeploymentGitOpsService.go @@ -0,0 +1,272 @@ +package appStoreDeploymentTool + +import ( + "errors" + "fmt" + bean2 "github.com/devtron-labs/devtron/api/bean" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/util" + appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" + "github.com/devtron-labs/devtron/pkg/appStore/deployment/adapter" + "github.com/devtron-labs/devtron/pkg/appStore/deployment/tool/bean" + appStoreDiscoverRepository "github.com/devtron-labs/devtron/pkg/appStore/discover/repository" + commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" + "github.com/google/go-github/github" + "github.com/microsoft/azure-devops-go-api/azuredevops" + "github.com/xanzy/go-gitlab" + "net/http" + "path" + "regexp" +) + +func (impl AppStoreDeploymentArgoCdServiceImpl) ParseGitRepoErrorResponse(err error) (bool, error) { + //update values yaml in chart + noTargetFound := false + if err != nil { + if errorResponse, ok := err.(*github.ErrorResponse); ok && errorResponse.Response.StatusCode == http.StatusNotFound { + impl.Logger.Errorw("no content found while updating git repo on github, do auto fix", "error", err) + noTargetFound = true + } + if errorResponse, ok := err.(azuredevops.WrappedError); ok && *errorResponse.StatusCode == http.StatusNotFound { + impl.Logger.Errorw("no content found while updating git repo on azure, do auto fix", "error", err) + noTargetFound = true + } + if errorResponse, ok := err.(*azuredevops.WrappedError); ok && *errorResponse.StatusCode == http.StatusNotFound { + impl.Logger.Errorw("no content found while updating git repo on azure, do auto fix", "error", err) + noTargetFound = true + } + if errorResponse, ok := err.(*gitlab.ErrorResponse); ok && errorResponse.Response.StatusCode == http.StatusNotFound { + impl.Logger.Errorw("no content found while updating git repo gitlab, do auto fix", "error", err) + noTargetFound = true + } + if err.Error() == util.BITBUCKET_REPO_NOT_FOUND_ERROR { + impl.Logger.Errorw("no content found while updating git repo bitbucket, do auto fix", "error", err) + noTargetFound = true + } + } + return noTargetFound, err +} + +func (impl AppStoreDeploymentArgoCdServiceImpl) GenerateManifestAndPerformGitOperations(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*bean.AppStoreGitOpsResponse, error) { + appStoreGitOpsResponse := &bean.AppStoreGitOpsResponse{} + manifest, err := impl.GenerateManifest(installAppVersionRequest) + if err != nil { + impl.Logger.Errorw("error in performing manifest and git operations", "err", err) + return nil, err + } + gitOpsResponse, err := impl.GitOpsOperations(manifest, installAppVersionRequest) + if err != nil { + impl.Logger.Errorw("error in performing gitops operation", "err", err) + return nil, err + } + installAppVersionRequest.GitHash = gitOpsResponse.GitHash + appStoreGitOpsResponse.ChartGitAttribute = gitOpsResponse.ChartGitAttribute + return appStoreGitOpsResponse, nil +} + +// GitOpsOperations handles all git operations for Helm App +func (impl AppStoreDeploymentArgoCdServiceImpl) GitOpsOperations(manifestResponse *bean.AppStoreManifestResponse, installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*bean.AppStoreGitOpsResponse, error) { + appStoreGitOpsResponse := &bean.AppStoreGitOpsResponse{} + chartGitAttribute, isNew, githash, err := impl.createGitOpsRepoAndPushChart(installAppVersionRequest, manifestResponse.ChartResponse.BuiltChartPath, manifestResponse.RequirementsConfig, manifestResponse.ValuesConfig) + if err != nil { + impl.Logger.Errorw("Error in pushing chart to git", "err", err) + return appStoreGitOpsResponse, err + } + space := regexp.MustCompile(`\s+`) + appStoreName := space.ReplaceAllString(installAppVersionRequest.AppName, "-") + + // Checking this is the first time chart has been pushed , if yes requirements.yaml has been already pushed with chart as there was sync-delay with github api. + // step-2 commit dependencies and values in git + if !isNew { + githash, err = impl.gitOpsRemoteOperationService.CommitRequirementsAndValues(appStoreName, chartGitAttribute.RepoUrl, manifestResponse.RequirementsConfig, manifestResponse.ValuesConfig) + if err != nil { + impl.Logger.Errorw("error in committing config to git", "err", err) + return appStoreGitOpsResponse, err + } + } + appStoreGitOpsResponse.ChartGitAttribute = chartGitAttribute + appStoreGitOpsResponse.GitHash = githash + return appStoreGitOpsResponse, nil +} + +func (impl AppStoreDeploymentArgoCdServiceImpl) GenerateManifest(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (manifestResponse *bean.AppStoreManifestResponse, err error) { + + manifestResponse = &bean.AppStoreManifestResponse{} + + ChartCreateResponse, err := impl.createChartProxyAndGetPath(installAppVersionRequest) + if err != nil { + impl.Logger.Errorw("Error in building chart while generating manifest", "err", err) + return manifestResponse, err + } + valuesConfig, dependencyConfig, err := impl.getValuesAndRequirementForGitConfig(installAppVersionRequest) + if err != nil { + impl.Logger.Errorw("error in fetching values and requirements.yaml config while generating manifest", "err", err) + return manifestResponse, err + } + + manifestResponse.ChartResponse = ChartCreateResponse + manifestResponse.ValuesConfig = valuesConfig + manifestResponse.RequirementsConfig = dependencyConfig + + return manifestResponse, nil +} + +// createGitOpsRepoAndPushChart is a wrapper for creating GitOps repo and pushing chart to created repo +func (impl AppStoreDeploymentArgoCdServiceImpl) createGitOpsRepoAndPushChart(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, builtChartPath string, requirementsConfig *util.ChartConfig, valuesConfig *util.ChartConfig) (*commonBean.ChartGitAttribute, bool, string, error) { + repoURL, isNew, err := impl.createGitOpsRepo(installAppVersionRequest) + if err != nil { + impl.Logger.Errorw("Error in creating gitops repo for ", "appName", installAppVersionRequest.AppName, "err", err) + return nil, false, "", err + } + pushChartToGitRequest := adapter.ParseChartGitPushRequest(installAppVersionRequest, repoURL, builtChartPath) + chartGitAttribute, commitHash, err := impl.gitOpsRemoteOperationService.PushChartToGitOpsRepoForHelmApp(pushChartToGitRequest, requirementsConfig, valuesConfig) + if err != nil { + impl.Logger.Errorw("error in pushing chart to git", "err", err) + return nil, false, "", err + } + return chartGitAttribute, isNew, commitHash, err +} + +// createGitOpsRepo creates a gitOps repo with readme +func (impl AppStoreDeploymentArgoCdServiceImpl) createGitOpsRepo(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (string, bool, error) { + if len(installAppVersionRequest.GitOpsRepoName) == 0 { + //here gitops repo will be the app name, to breaking the mono repo structure + gitOpsRepoName := impl.gitOpsConfigReadService.GetGitOpsRepoName(installAppVersionRequest.AppName) + installAppVersionRequest.GitOpsRepoName = gitOpsRepoName + } + bitbucketMetadata, err := impl.gitOpsConfigReadService.GetBitbucketMetadata() + if err != nil { + impl.Logger.Errorw("error in getting bitbucket metadata", "err", err) + return "", false, err + } + //getting user name & emailId for commit author data + gitRepoRequest := &bean2.GitOpsConfigDto{ + GitRepoName: installAppVersionRequest.GitOpsRepoName, + Description: "helm chart for " + installAppVersionRequest.GitOpsRepoName, + BitBucketWorkspaceId: bitbucketMetadata.BitBucketWorkspaceId, + BitBucketProjectKey: bitbucketMetadata.BitBucketProjectKey, + } + repoUrl, isNew, err := impl.gitOpsRemoteOperationService.CreateRepository(gitRepoRequest, installAppVersionRequest.UserId) + if err != nil { + impl.Logger.Errorw("error in creating git project", "name", installAppVersionRequest.GitOpsRepoName, "err", err) + return "", false, err + } + return repoUrl, isNew, err +} + +func (impl AppStoreDeploymentArgoCdServiceImpl) updateValuesYamlInGit(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*appStoreBean.InstallAppVersionDTO, error) { + valuesString, err := impl.appStoreDeploymentCommonService.GetValuesString(installAppVersionRequest.AppStoreName, installAppVersionRequest.ValuesOverrideYaml) + if err != nil { + impl.Logger.Errorw("error in getting values string", "err", err) + return nil, err + } + + valuesGitConfig, err := impl.getGitCommitConfig(installAppVersionRequest, valuesString, appStoreBean.VALUES_YAML_FILE) + if err != nil { + impl.Logger.Errorw("error in getting git commit config", "err", err) + } + + commitHash, _, err := impl.gitOpsRemoteOperationService.CommitValues(valuesGitConfig) + if err != nil { + impl.Logger.Errorw("error in git commit", "err", err) + return installAppVersionRequest, errors.New(pipelineConfig.TIMELINE_STATUS_GIT_COMMIT_FAILED) + } + //update timeline status for git commit state + installAppVersionRequest.GitHash = commitHash + return installAppVersionRequest, nil +} + +func (impl AppStoreDeploymentArgoCdServiceImpl) updateRequirementYamlInGit(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, appStoreAppVersion *appStoreDiscoverRepository.AppStoreApplicationVersion) error { + requirementsString, err := impl.appStoreDeploymentCommonService.GetRequirementsString(appStoreAppVersion.Id) + if err != nil { + impl.Logger.Errorw("error in getting requirements string", "err", err) + return err + } + + requirementsGitConfig, err := impl.getGitCommitConfig(installAppVersionRequest, requirementsString, appStoreBean.REQUIREMENTS_YAML_FILE) + if err != nil { + impl.Logger.Errorw("error in getting git commit config", "err", err) + return err + } + + _, _, err = impl.gitOpsRemoteOperationService.CommitValues(requirementsGitConfig) + if err != nil { + impl.Logger.Errorw("error in values commit", "err", err) + return errors.New(pipelineConfig.TIMELINE_STATUS_GIT_COMMIT_FAILED) + } + + return nil +} + +// createChartProxyAndGetPath parse chart in local directory and returns path of local dir and values.yaml +func (impl AppStoreDeploymentArgoCdServiceImpl) createChartProxyAndGetPath(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*util.ChartCreateResponse, error) { + template := appStoreBean.CHART_PROXY_TEMPLATE + chartPath := path.Join(appStoreBean.RefChartProxyDirPath, template) + chartCreateRequest := adapter.ParseChartCreateRequest(installAppVersionRequest, chartPath) + chartCreateResponse, err := impl.appStoreDeploymentCommonService.CreateChartProxyAndGetPath(chartCreateRequest) + if err != nil { + impl.Logger.Errorw("Error in building chart proxy", "err", err) + return chartCreateResponse, err + } + return chartCreateResponse, nil + +} + +// getGitCommitConfig will return util.ChartConfig (git commit config) for GitOps +func (impl AppStoreDeploymentArgoCdServiceImpl) getGitCommitConfig(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, fileString string, filename string) (*util.ChartConfig, error) { + appStoreAppVersion, err := impl.appStoreApplicationVersionRepository.FindById(installAppVersionRequest.AppStoreVersion) + if err != nil { + impl.Logger.Errorw("fetching error", "err", err) + return nil, err + } + environment, err := impl.environmentRepository.FindById(installAppVersionRequest.EnvironmentId) + if err != nil { + impl.Logger.Errorw("fetching error", "err", err) + return nil, err + } + + argocdAppName := installAppVersionRequest.AppName + "-" + environment.Name + gitOpsRepoName := impl.gitOpsConfigReadService.GetGitOpsRepoName(installAppVersionRequest.AppName) + userEmailId, userName := impl.gitOpsConfigReadService.GetUserEmailIdAndNameForGitOpsCommit(installAppVersionRequest.UserId) + YamlConfig := &util.ChartConfig{ + FileName: filename, + FileContent: fileString, + ChartName: installAppVersionRequest.AppName, + ChartLocation: argocdAppName, + ChartRepoName: gitOpsRepoName, + ReleaseMessage: fmt.Sprintf("release-%d-env-%d ", appStoreAppVersion.Id, environment.Id), + UserEmailId: userEmailId, + UserName: userName, + } + return YamlConfig, nil +} + +// getValuesAndRequirementForGitConfig will return chart values(*util.ChartConfig) and requirements(*util.ChartConfig) for git commit +func (impl AppStoreDeploymentArgoCdServiceImpl) getValuesAndRequirementForGitConfig(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*util.ChartConfig, *util.ChartConfig, error) { + appStoreAppVersion, err := impl.appStoreApplicationVersionRepository.FindById(installAppVersionRequest.AppStoreVersion) + if err != nil { + impl.Logger.Errorw("fetching error", "err", err) + return nil, nil, err + } + values, err := impl.appStoreDeploymentCommonService.GetValuesString(appStoreAppVersion.AppStore.Name, installAppVersionRequest.ValuesOverrideYaml) + if err != nil { + impl.Logger.Errorw("error in getting values fot installedAppVersionRequest", "err", err) + return nil, nil, err + } + dependency, err := impl.appStoreDeploymentCommonService.GetRequirementsString(installAppVersionRequest.AppStoreVersion) + if err != nil { + impl.Logger.Errorw("error in getting dependency array fot installedAppVersionRequest", "err", err) + return nil, nil, err + } + valuesConfig, err := impl.getGitCommitConfig(installAppVersionRequest, values, appStoreBean.VALUES_YAML_FILE) + if err != nil { + impl.Logger.Errorw("error in creating values config for git", "err", err) + return nil, nil, err + } + RequirementConfig, err := impl.getGitCommitConfig(installAppVersionRequest, dependency, appStoreBean.REQUIREMENTS_YAML_FILE) + if err != nil { + impl.Logger.Errorw("error in creating dependency config for git", "err", err) + return nil, nil, err + } + return valuesConfig, RequirementConfig, nil +} diff --git a/pkg/appStore/deployment/tool/AppStoreDeploymentHelmService.go b/pkg/appStore/deployment/tool/AppStoreDeploymentHelmService.go index e1384780ed..4f4d28c7a8 100644 --- a/pkg/appStore/deployment/tool/AppStoreDeploymentHelmService.go +++ b/pkg/appStore/deployment/tool/AppStoreDeploymentHelmService.go @@ -3,16 +3,17 @@ package appStoreDeploymentTool import ( "context" "errors" + bean2 "github.com/devtron-labs/devtron/api/helm-app/bean" + "github.com/devtron-labs/devtron/api/helm-app/gRPC" + client "github.com/devtron-labs/devtron/api/helm-app/service" repository2 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" + "github.com/devtron-labs/devtron/pkg/appStore/deployment/tool/bean" commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" - "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote" "time" - client "github.com/devtron-labs/devtron/api/helm-app" openapi "github.com/devtron-labs/devtron/api/helm-app/openapiClient" "github.com/devtron-labs/devtron/internal/util" appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" - appStoreDeploymentCommon "github.com/devtron-labs/devtron/pkg/appStore/deployment/common" "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" appStoreDiscoverRepository "github.com/devtron-labs/devtron/pkg/appStore/discover/repository" "github.com/go-pg/pg" @@ -25,7 +26,7 @@ type AppStoreDeploymentHelmService interface { InstallApp(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, chartGitAttr *commonBean.ChartGitAttribute, ctx context.Context, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, error) DeleteInstalledApp(ctx context.Context, appName string, environmentName string, installAppVersionRequest *appStoreBean.InstallAppVersionDTO, installedApps *repository.InstalledApps, dbTransaction *pg.Tx) error RollbackRelease(ctx context.Context, installedApp *appStoreBean.InstallAppVersionDTO, deploymentVersion int32, tx *pg.Tx) (*appStoreBean.InstallAppVersionDTO, bool, error) - GetDeploymentHistory(ctx context.Context, installedApp *appStoreBean.InstallAppVersionDTO) (*client.HelmAppDeploymentHistory, error) + GetDeploymentHistory(ctx context.Context, installedApp *appStoreBean.InstallAppVersionDTO) (*gRPC.HelmAppDeploymentHistory, error) GetDeploymentHistoryInfo(ctx context.Context, installedApp *appStoreBean.InstallAppVersionDTO, version int32) (*openapi.HelmAppDeploymentManifestDetail, error) DeleteDeploymentApp(ctx context.Context, appName string, environmentName string, installAppVersionRequest *appStoreBean.InstallAppVersionDTO) error UpdateChartInfo(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, ChartGitAttribute *commonBean.ChartGitAttribute, installedAppVersionHistoryId int, ctx context.Context) error @@ -35,28 +36,24 @@ type AppStoreDeploymentHelmServiceImpl struct { Logger *zap.SugaredLogger helmAppService client.HelmAppService appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository - helmAppClient client.HelmAppClient - installedAppRepository repository.InstalledAppRepository - appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService - OCIRegistryConfigRepository repository2.OCIRegistryConfigRepository - gitOpsRemoteOperationService remote.GitOpsRemoteOperationService + // TODO fix me next + helmAppClient gRPC.HelmAppClient // TODO refactoring: use HelmAppService instead + installedAppRepository repository.InstalledAppRepository + OCIRegistryConfigRepository repository2.OCIRegistryConfigRepository } func NewAppStoreDeploymentHelmServiceImpl(logger *zap.SugaredLogger, helmAppService client.HelmAppService, appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository, - helmAppClient client.HelmAppClient, - installedAppRepository repository.InstalledAppRepository, appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService, - OCIRegistryConfigRepository repository2.OCIRegistryConfigRepository, - gitOpsRemoteOperationService remote.GitOpsRemoteOperationService) *AppStoreDeploymentHelmServiceImpl { + helmAppClient gRPC.HelmAppClient, + installedAppRepository repository.InstalledAppRepository, + OCIRegistryConfigRepository repository2.OCIRegistryConfigRepository) *AppStoreDeploymentHelmServiceImpl { return &AppStoreDeploymentHelmServiceImpl{ Logger: logger, helmAppService: helmAppService, appStoreApplicationVersionRepository: appStoreApplicationVersionRepository, helmAppClient: helmAppClient, installedAppRepository: installedAppRepository, - appStoreDeploymentCommonService: appStoreDeploymentCommonService, OCIRegistryConfigRepository: OCIRegistryConfigRepository, - gitOpsRemoteOperationService: gitOpsRemoteOperationService, } } @@ -77,8 +74,8 @@ func (impl AppStoreDeploymentHelmServiceImpl) InstallApp(installAppVersionReques return installAppVersionRequest, err } var IsOCIRepo bool - var registryCredential *client.RegistryCredential - var chartRepository *client.ChartRepository + var registryCredential *gRPC.RegistryCredential + var chartRepository *gRPC.ChartRepository dockerRegistryId := appStoreAppVersion.AppStore.DockerArtifactStoreId if dockerRegistryId != "" { ociRegistryConfigs, err := impl.OCIRegistryConfigRepository.FindByDockerRegistryId(dockerRegistryId) @@ -94,7 +91,7 @@ func (impl AppStoreDeploymentHelmServiceImpl) InstallApp(installAppVersionReques } } IsOCIRepo = true - registryCredential = &client.RegistryCredential{ + registryCredential = &gRPC.RegistryCredential{ RegistryUrl: appStoreAppVersion.AppStore.DockerArtifactStore.RegistryURL, Username: appStoreAppVersion.AppStore.DockerArtifactStore.Username, Password: appStoreAppVersion.AppStore.DockerArtifactStore.Password, @@ -106,19 +103,19 @@ func (impl AppStoreDeploymentHelmServiceImpl) InstallApp(installAppVersionReques IsPublic: ociRegistryConfig.IsPublic, } } else { - chartRepository = &client.ChartRepository{ + chartRepository = &gRPC.ChartRepository{ Name: appStoreAppVersion.AppStore.ChartRepo.Name, Url: appStoreAppVersion.AppStore.ChartRepo.Url, Username: appStoreAppVersion.AppStore.ChartRepo.UserName, Password: appStoreAppVersion.AppStore.ChartRepo.Password, } } - installReleaseRequest := &client.InstallReleaseRequest{ + installReleaseRequest := &gRPC.InstallReleaseRequest{ ChartName: appStoreAppVersion.Name, ChartVersion: appStoreAppVersion.Version, ValuesYaml: installAppVersionRequest.ValuesOverrideYaml, ChartRepository: chartRepository, - ReleaseIdentifier: &client.ReleaseIdentifier{ + ReleaseIdentifier: &gRPC.ReleaseIdentifier{ ReleaseNamespace: installAppVersionRequest.Namespace, ReleaseName: installAppVersionRequest.AppName, }, @@ -198,23 +195,13 @@ func (impl AppStoreDeploymentHelmServiceImpl) RollbackRelease(ctx context.Contex return installedApp, success, nil } -func (impl *AppStoreDeploymentHelmServiceImpl) GetDeploymentHistory(ctx context.Context, installedApp *appStoreBean.InstallAppVersionDTO) (*client.HelmAppDeploymentHistory, error) { - helmAppIdeltifier := &client.AppIdentifier{ +func (impl *AppStoreDeploymentHelmServiceImpl) GetDeploymentHistory(ctx context.Context, installedApp *appStoreBean.InstallAppVersionDTO) (*gRPC.HelmAppDeploymentHistory, error) { + helmAppIdentifier := &client.AppIdentifier{ ClusterId: installedApp.ClusterId, Namespace: installedApp.Namespace, ReleaseName: installedApp.AppName, } - config, err := impl.helmAppService.GetClusterConf(helmAppIdeltifier.ClusterId) - if err != nil { - impl.Logger.Errorw("error in fetching cluster detail", "err", err) - return nil, err - } - req := &client.AppDetailRequest{ - ClusterConfig: config, - Namespace: helmAppIdeltifier.Namespace, - ReleaseName: helmAppIdeltifier.ReleaseName, - } - history, err := impl.helmAppClient.GetDeploymentHistory(ctx, req) + history, err := impl.helmAppService.GetDeploymentHistory(ctx, helmAppIdentifier) return history, err } @@ -230,8 +217,8 @@ func (impl *AppStoreDeploymentHelmServiceImpl) GetDeploymentHistoryInfo(ctx cont return nil, err } - req := &client.DeploymentDetailRequest{ - ReleaseIdentifier: &client.ReleaseIdentifier{ + req := &gRPC.DeploymentDetailRequest{ + ReleaseIdentifier: &gRPC.ReleaseIdentifier{ ClusterConfig: config, ReleaseName: helmAppIdeltifier.ReleaseName, ReleaseNamespace: helmAppIdeltifier.Namespace, @@ -254,36 +241,7 @@ func (impl *AppStoreDeploymentHelmServiceImpl) GetDeploymentHistoryInfo(ctx cont return response, nil } -func (impl *AppStoreDeploymentHelmServiceImpl) GetGitOpsRepoName(appName string, environmentName string) (string, error) { - return "", errors.New("method GetGitOpsRepoName not implemented") -} - -func (impl *AppStoreDeploymentHelmServiceImpl) UpdateValuesDependencies(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) error { - appStoreAppVersion, err := impl.appStoreApplicationVersionRepository.FindById(installAppVersionRequest.AppStoreVersion) - if err != nil { - impl.Logger.Errorw("fetching error", "err", err) - return err - } - valuesString, err := impl.appStoreDeploymentCommonService.GetValuesString(appStoreAppVersion.Name, installAppVersionRequest.ValuesOverrideYaml) - if err != nil { - impl.Logger.Errorw("error in building requirements config for helm app", "err", err) - return err - } - valuesGitConfig, err := impl.appStoreDeploymentCommonService.GetGitCommitConfig(installAppVersionRequest, valuesString, appStoreBean.VALUES_YAML_FILE) - if err != nil { - impl.Logger.Errorw("error in getting git config for helm app", "err", err) - return err - } - _, _, err = impl.gitOpsRemoteOperationService.CommitValues(valuesGitConfig) - if err != nil { - impl.Logger.Errorw("error in committing config to git for helm app", "err", err) - return err - } - return nil -} - func (impl *AppStoreDeploymentHelmServiceImpl) updateApplicationWithChartInfo(ctx context.Context, installedAppId int, appStoreApplicationVersionId int, valuesOverrideYaml string, installAppVersionHistoryId int) error { - installedApp, err := impl.installedAppRepository.GetInstalledApp(installedAppId) if err != nil { impl.Logger.Errorw("error in getting in installedApp", "installedAppId", installedAppId, "err", err) @@ -295,8 +253,8 @@ func (impl *AppStoreDeploymentHelmServiceImpl) updateApplicationWithChartInfo(ct return err } var IsOCIRepo bool - var registryCredential *client.RegistryCredential - var chartRepository *client.ChartRepository + var registryCredential *gRPC.RegistryCredential + var chartRepository *gRPC.ChartRepository dockerRegistryId := appStoreApplicationVersion.AppStore.DockerArtifactStoreId if dockerRegistryId != "" { ociRegistryConfigs, err := impl.OCIRegistryConfigRepository.FindByDockerRegistryId(dockerRegistryId) @@ -312,7 +270,7 @@ func (impl *AppStoreDeploymentHelmServiceImpl) updateApplicationWithChartInfo(ct } } IsOCIRepo = true - registryCredential = &client.RegistryCredential{ + registryCredential = &gRPC.RegistryCredential{ RegistryUrl: appStoreApplicationVersion.AppStore.DockerArtifactStore.RegistryURL, Username: appStoreApplicationVersion.AppStore.DockerArtifactStore.Username, Password: appStoreApplicationVersion.AppStore.DockerArtifactStore.Password, @@ -324,7 +282,7 @@ func (impl *AppStoreDeploymentHelmServiceImpl) updateApplicationWithChartInfo(ct IsPublic: ociRegistryConfig.IsPublic, } } else { - chartRepository = &client.ChartRepository{ + chartRepository = &gRPC.ChartRepository{ Name: appStoreApplicationVersion.AppStore.ChartRepo.Name, Url: appStoreApplicationVersion.AppStore.ChartRepo.Url, Username: appStoreApplicationVersion.AppStore.ChartRepo.UserName, @@ -332,10 +290,10 @@ func (impl *AppStoreDeploymentHelmServiceImpl) updateApplicationWithChartInfo(ct } } - updateReleaseRequest := &client.UpdateApplicationWithChartInfoRequestDto{ - InstallReleaseRequest: &client.InstallReleaseRequest{ + updateReleaseRequest := &bean2.UpdateApplicationWithChartInfoRequestDto{ + InstallReleaseRequest: &gRPC.InstallReleaseRequest{ ValuesYaml: valuesOverrideYaml, - ReleaseIdentifier: &client.ReleaseIdentifier{ + ReleaseIdentifier: &gRPC.ReleaseIdentifier{ ReleaseNamespace: installedApp.Environment.Namespace, ReleaseName: installedApp.App.AppName, }, @@ -346,7 +304,7 @@ func (impl *AppStoreDeploymentHelmServiceImpl) updateApplicationWithChartInfo(ct IsOCIRepo: IsOCIRepo, InstallAppVersionHistoryId: int32(installAppVersionHistoryId), }, - SourceAppType: client.SOURCE_HELM_APP, + SourceAppType: bean2.SOURCE_HELM_APP, } res, err := impl.helmAppService.UpdateApplicationWithChartInfo(ctx, installedApp.Environment.ClusterId, updateReleaseRequest) if err != nil { @@ -358,16 +316,21 @@ func (impl *AppStoreDeploymentHelmServiceImpl) updateApplicationWithChartInfo(ct } return nil } + +func (impl *AppStoreDeploymentHelmServiceImpl) GetAcdAppGitOpsRepoName(appName string, environmentName string) (string, error) { + return "", errors.New("method GetGitOpsRepoName not implemented") +} + func (impl *AppStoreDeploymentHelmServiceImpl) DeleteDeploymentApp(ctx context.Context, appName string, environmentName string, installAppVersionRequest *appStoreBean.InstallAppVersionDTO) error { - return nil + return errors.New("this is not implemented") } func (impl *AppStoreDeploymentHelmServiceImpl) SaveTimelineForACDHelmApps(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, status string, statusDetail string, statusTime time.Time, tx *pg.Tx) error { - return nil + return errors.New("this is not implemented") } func (impl *AppStoreDeploymentHelmServiceImpl) UpdateInstalledAppAndPipelineStatusForFailedDeploymentStatus(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, triggeredAt time.Time, err error) error { - return nil + return errors.New("this is not implemented") } // TODO: Need to refactor this,refer below reason @@ -376,3 +339,23 @@ func (impl *AppStoreDeploymentHelmServiceImpl) UpdateInstalledAppAndPipelineStat func (impl *AppStoreDeploymentHelmServiceImpl) UpdateAndSyncACDApps(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, ChartGitAttribute *commonBean.ChartGitAttribute, isMonoRepoMigrationRequired bool, ctx context.Context, tx *pg.Tx) error { return errors.New("this is not implemented") } + +func (impl *AppStoreDeploymentHelmServiceImpl) UpdateValuesDependencies(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) error { + return errors.New("this is not implemented") +} + +func (impl *AppStoreDeploymentHelmServiceImpl) ParseGitRepoErrorResponse(err error) (bool, error) { + return false, errors.New("this is not implemented") +} + +func (impl *AppStoreDeploymentHelmServiceImpl) GitOpsOperations(manifestResponse *bean.AppStoreManifestResponse, installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*bean.AppStoreGitOpsResponse, error) { + return nil, errors.New("this is not implemented") +} + +func (impl *AppStoreDeploymentHelmServiceImpl) GenerateManifestAndPerformGitOperations(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*bean.AppStoreGitOpsResponse, error) { + return nil, errors.New("this is not implemented") +} + +func (impl *AppStoreDeploymentHelmServiceImpl) GenerateManifest(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (manifestResponse *bean.AppStoreManifestResponse, err error) { + return nil, errors.New("this is not implemented") +} diff --git a/pkg/appStore/deployment/tool/bean/bean.go b/pkg/appStore/deployment/tool/bean/bean.go new file mode 100644 index 0000000000..9820820779 --- /dev/null +++ b/pkg/appStore/deployment/tool/bean/bean.go @@ -0,0 +1,17 @@ +package bean + +import ( + "github.com/devtron-labs/devtron/internal/util" + commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" +) + +type AppStoreManifestResponse struct { + ChartResponse *util.ChartCreateResponse + ValuesConfig *util.ChartConfig + RequirementsConfig *util.ChartConfig +} + +type AppStoreGitOpsResponse struct { + ChartGitAttribute *commonBean.ChartGitAttribute + GitHash string +} diff --git a/pkg/bulkAction/BulkUpdateService.go b/pkg/bulkAction/BulkUpdateService.go index 865b7fa171..a015279dec 100644 --- a/pkg/bulkAction/BulkUpdateService.go +++ b/pkg/bulkAction/BulkUpdateService.go @@ -9,8 +9,8 @@ import ( pubsub "github.com/devtron-labs/common-lib/pubsub-lib" "github.com/devtron-labs/common-lib/pubsub-lib/model" "github.com/devtron-labs/devtron/api/bean" - client "github.com/devtron-labs/devtron/api/helm-app" openapi "github.com/devtron-labs/devtron/api/helm-app/openapiClient" + client "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/client/argocdServer/application" "github.com/devtron-labs/devtron/internal/sql/models" "github.com/devtron-labs/devtron/internal/sql/repository/app" diff --git a/pkg/clusterTerminalAccess/UserTerminalAccessService.go b/pkg/clusterTerminalAccess/UserTerminalAccessService.go index d19162d251..fc14c5db61 100644 --- a/pkg/clusterTerminalAccess/UserTerminalAccessService.go +++ b/pkg/clusterTerminalAccess/UserTerminalAccessService.go @@ -7,7 +7,7 @@ import ( "fmt" "github.com/caarlos0/env/v6" k8s2 "github.com/devtron-labs/common-lib/utils/k8s" - client "github.com/devtron-labs/devtron/api/helm-app" + client "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/internal/sql/models" "github.com/devtron-labs/devtron/internal/sql/repository" utils1 "github.com/devtron-labs/devtron/pkg/clusterTerminalAccess/clusterTerminalUtils" diff --git a/pkg/deployment/gitOps/remote/GitOpsRemoteOperationService.go b/pkg/deployment/gitOps/remote/GitOpsRemoteOperationService.go index 18369cccf3..09b0cf01d8 100644 --- a/pkg/deployment/gitOps/remote/GitOpsRemoteOperationService.go +++ b/pkg/deployment/gitOps/remote/GitOpsRemoteOperationService.go @@ -9,11 +9,9 @@ import ( "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote/bean" dirCopy "github.com/otiai10/copy" "go.uber.org/zap" - "k8s.io/helm/pkg/proto/hapi/chart" "os" "path/filepath" "regexp" - "sigs.k8s.io/yaml" "time" ) @@ -23,12 +21,14 @@ type GitOpsRemoteOperationService interface { PushChartToGitRepo(gitOpsRepoName, referenceTemplate, version, tempReferenceTemplateDir string, repoUrl string, userId int32) (err error) CreateReadmeInGitRepo(gitOpsRepoName string, userId int32) error - CreateChartProxy(chartMetaData *chart.Metadata, refChartLocation string, envName string, - chartProxyReq *bean.ChartProxyReqDto) (string, *commonBean.ChartGitAttribute, error) GitPull(clonedDir string, repoUrl string, appStoreName string) error CommitValues(chartGitAttr *util.ChartConfig) (commitHash string, commitTime time.Time, err error) + CommitRequirementsAndValues(appStoreName, repoUrl string, requirementsConfig *util.ChartConfig, + valuesConfig *util.ChartConfig) (gitHash string, err error) CreateRepository(dto *bean2.GitOpsConfigDto, userId int32) (string, bool, error) GetRepoUrlByRepoName(repoName string) (string, error) + PushChartToGitOpsRepoForHelmApp(PushChartToGitRequest *bean.PushChartToGitRequestDTO, + requirementsConfig *util.ChartConfig, valuesConfig *util.ChartConfig) (*commonBean.ChartGitAttribute, string, error) } type GitOpsRemoteOperationServiceImpl struct { @@ -43,6 +43,7 @@ func NewGitOpsRemoteOperationServiceImpl(logger *zap.SugaredLogger, gitFactory * chartTemplateService util.ChartTemplateService) *GitOpsRemoteOperationServiceImpl { return &GitOpsRemoteOperationServiceImpl{ logger: logger, + gitFactory: gitFactory, gitOpsConfigReadService: gitOpsConfigReadService, chartTemplateService: chartTemplateService, } @@ -179,47 +180,6 @@ func (impl *GitOpsRemoteOperationServiceImpl) CreateReadmeInGitRepo(gitOpsRepoNa return nil } -func (impl *GitOpsRemoteOperationServiceImpl) CreateChartProxy(chartMetaData *chart.Metadata, refChartLocation string, envName string, - chartProxyReq *bean.ChartProxyReqDto) (string, *commonBean.ChartGitAttribute, error) { - chartMetaData.ApiVersion = "v2" // ensure always v2 - dir := impl.chartTemplateService.GetDir() - chartDir := filepath.Join(util.ChartWorkingDirPath, dir) - impl.logger.Debugw("chart dir ", "chart", chartMetaData.Name, "dir", chartDir) - err := os.MkdirAll(chartDir, os.ModePerm) //hack for concurrency handling - if err != nil { - impl.logger.Errorw("err in creating dir", "dir", chartDir, "err", err) - return "", nil, err - } - defer impl.chartTemplateService.CleanDir(chartDir) - err = dirCopy.Copy(refChartLocation, chartDir) - - if err != nil { - impl.logger.Errorw("error in copying chart for app", "app", chartMetaData.Name, "error", err) - return "", nil, err - } - archivePath, valuesYaml, err := impl.chartTemplateService.PackageChart(chartDir, chartMetaData) - if err != nil { - impl.logger.Errorw("error in creating archive", "err", err) - return "", nil, err - } - - chartGitAttr, err := impl.createAndPushToGitChartProxy(chartMetaData.Name, chartDir, envName, chartProxyReq) - if err != nil { - impl.logger.Errorw("error in pushing chart to git ", "path", archivePath, "err", err) - return "", nil, err - } - if valuesYaml == "" { - valuesYaml = "{}" - } else { - valuesYamlByte, err := yaml.YAMLToJSON([]byte(valuesYaml)) - if err != nil { - return "", nil, err - } - valuesYaml = string(valuesYamlByte) - } - return valuesYaml, chartGitAttr, nil -} - func (impl *GitOpsRemoteOperationServiceImpl) createAndPushToGitChartProxy(appStoreName, tmpChartLocation string, envName string, chartProxyReq *bean.ChartProxyReqDto) (chartGitAttribute *commonBean.ChartGitAttribute, err error) { //baseTemplateName replace whitespace @@ -306,7 +266,10 @@ func (impl *GitOpsRemoteOperationServiceImpl) createAndPushToGitChartProxy(appSt } func (impl *GitOpsRemoteOperationServiceImpl) GitPull(clonedDir string, repoUrl string, appStoreName string) error { - err := impl.gitFactory.GitService.Pull(clonedDir) //TODO check for local repo exists before clone + //TODO refactoring: remove invalid param appStoreName + //TODO check for local repo exists before clone + //TODO verify remote has repoUrl; or delete and clone + err := impl.gitFactory.GitService.Pull(clonedDir) if err != nil { impl.logger.Errorw("error in pulling git", "clonedDir", clonedDir, "err", err) _, err := impl.gitFactory.GitService.Clone(repoUrl, appStoreName) @@ -373,3 +336,96 @@ func (impl *GitOpsRemoteOperationServiceImpl) GetRepoUrlByRepoName(repoName stri } return repoUrl, nil } + +// TODO refactoring: Make a common method for both PushChartToGitRepo and PushChartToGitOpsRepoForHelmApp +// PushChartToGitOpsRepoForHelmApp pushes built chart to GitOps repo (Specific implementation for Helm Apps) +func (impl *GitOpsRemoteOperationServiceImpl) PushChartToGitOpsRepoForHelmApp(PushChartToGitRequest *bean.PushChartToGitRequestDTO, requirementsConfig *util.ChartConfig, valuesConfig *util.ChartConfig) (*commonBean.ChartGitAttribute, string, error) { + space := regexp.MustCompile(`\s+`) + appStoreName := space.ReplaceAllString(PushChartToGitRequest.ChartAppStoreName, "-") + chartDir := fmt.Sprintf("%s-%s", PushChartToGitRequest.AppName, impl.chartTemplateService.GetDir()) + clonedDir := impl.gitFactory.GitService.GetCloneDirectory(chartDir) + if _, err := os.Stat(clonedDir); os.IsNotExist(err) { + clonedDir, err = impl.gitFactory.GitService.Clone(PushChartToGitRequest.RepoURL, chartDir) + if err != nil { + impl.logger.Errorw("error in cloning repo", "url", PushChartToGitRequest.RepoURL, "err", err) + return nil, "", err + } + } else { + err = impl.GitPull(clonedDir, PushChartToGitRequest.RepoURL, appStoreName) + if err != nil { + return nil, "", err + } + } + acdAppName := fmt.Sprintf("%s-%s", PushChartToGitRequest.AppName, PushChartToGitRequest.EnvName) + dir := filepath.Join(clonedDir, acdAppName) + err := os.MkdirAll(dir, os.ModePerm) + if err != nil { + impl.logger.Errorw("error in making dir", "err", err) + return nil, "", err + } + err = dirCopy.Copy(PushChartToGitRequest.TempChartRefDir, dir) + if err != nil { + impl.logger.Errorw("error copying dir", "err", err) + return nil, "", err + } + err = impl.chartTemplateService.AddConfigFileToChart(requirementsConfig, dir, clonedDir) + if err != nil { + impl.logger.Errorw("error in adding requirements.yaml to chart", "err", err, "appName", PushChartToGitRequest.AppName) + return nil, "", err + } + err = impl.chartTemplateService.AddConfigFileToChart(valuesConfig, dir, clonedDir) + if err != nil { + impl.logger.Errorw("error in adding values.yaml to chart", "err", err, "appName", PushChartToGitRequest.AppName) + return nil, "", err + } + userEmailId, userName := impl.gitOpsConfigReadService.GetUserEmailIdAndNameForGitOpsCommit(PushChartToGitRequest.UserId) + commit, err := impl.gitFactory.GitService.CommitAndPushAllChanges(clonedDir, "first commit", userName, userEmailId) + if err != nil { + impl.logger.Errorw("error in pushing git", "err", err) + impl.logger.Warn("re-trying, taking pull and then push again") + err = impl.GitPull(clonedDir, PushChartToGitRequest.RepoURL, acdAppName) + if err != nil { + impl.logger.Errorw("error in git pull", "err", err, "appName", acdAppName) + return nil, "", err + } + err = dirCopy.Copy(PushChartToGitRequest.TempChartRefDir, dir) + if err != nil { + impl.logger.Errorw("error copying dir", "err", err) + return nil, "", err + } + commit, err = impl.gitFactory.GitService.CommitAndPushAllChanges(clonedDir, "first commit", userName, userEmailId) + if err != nil { + impl.logger.Errorw("error in pushing git", "err", err) + return nil, "", err + } + } + impl.logger.Debugw("template committed", "url", PushChartToGitRequest.RepoURL, "commit", commit) + defer impl.chartTemplateService.CleanDir(clonedDir) + return &commonBean.ChartGitAttribute{RepoUrl: PushChartToGitRequest.RepoURL, ChartLocation: acdAppName}, commit, err +} + +func (impl *GitOpsRemoteOperationServiceImpl) CommitRequirementsAndValues(appStoreName, repoUrl string, requirementsConfig *util.ChartConfig, valuesConfig *util.ChartConfig) (gitHash string, err error) { + clonedDir := impl.gitFactory.GitWorkingDir + "" + appStoreName + _, _, err = impl.CommitValues(requirementsConfig) + if err != nil { + impl.logger.Errorw("error in committing dependency config to git", "err", err) + return gitHash, err + } + err = impl.GitPull(clonedDir, repoUrl, appStoreName) + if err != nil { + impl.logger.Errorw("error in git pull", "err", err) + return gitHash, err + } + + gitHash, _, err = impl.CommitValues(valuesConfig) + if err != nil { + impl.logger.Errorw("error in committing values config to git", "err", err) + return gitHash, err + } + err = impl.GitPull(clonedDir, repoUrl, appStoreName) + if err != nil { + impl.logger.Errorw("error in git pull", "err", err) + return gitHash, err + } + return gitHash, nil +} diff --git a/pkg/deployment/gitOps/remote/bean/bean.go b/pkg/deployment/gitOps/remote/bean/bean.go index 361b0ec78c..7ac1fb0a15 100644 --- a/pkg/deployment/gitOps/remote/bean/bean.go +++ b/pkg/deployment/gitOps/remote/bean/bean.go @@ -5,3 +5,12 @@ type ChartProxyReqDto struct { AppName string `json:"appName,omitempty"` UserId int32 `json:"-"` } + +type PushChartToGitRequestDTO struct { + AppName string + EnvName string + ChartAppStoreName string + RepoURL string + TempChartRefDir string + UserId int32 +} diff --git a/pkg/generateManifest/DeployementTemplateService.go b/pkg/generateManifest/DeployementTemplateService.go index b35924856a..ba601ff6f6 100644 --- a/pkg/generateManifest/DeployementTemplateService.go +++ b/pkg/generateManifest/DeployementTemplateService.go @@ -4,7 +4,9 @@ import ( "context" "fmt" "github.com/devtron-labs/common-lib/utils/k8s" - client "github.com/devtron-labs/devtron/api/helm-app" + "github.com/devtron-labs/devtron/api/helm-app/bean" + "github.com/devtron-labs/devtron/api/helm-app/gRPC" + client "github.com/devtron-labs/devtron/api/helm-app/service" openapi2 "github.com/devtron-labs/devtron/api/openapi/openapiClient" "github.com/devtron-labs/devtron/internal/sql/repository" appRepository "github.com/devtron-labs/devtron/internal/sql/repository/app" @@ -46,14 +48,14 @@ const ( Manifest RequestDataMode = 2 ) -var ChartRepository = &client.ChartRepository{ +var ChartRepository = &gRPC.ChartRepository{ Name: "repo", Url: "http://localhost:8080/", Username: "admin", Password: "password", } -var ReleaseIdentifier = &client.ReleaseIdentifier{ +var ReleaseIdentifier = &gRPC.ReleaseIdentifier{ ReleaseNamespace: "devtron-demo", ReleaseName: "release-name", } @@ -79,7 +81,7 @@ type DeploymentTemplateServiceImpl struct { chartRepository chartRepoRepository.ChartRepository chartTemplateServiceImpl util.ChartTemplateService K8sUtil *k8s.K8sUtil - helmAppClient client.HelmAppClient + helmAppClient gRPC.HelmAppClient propertiesConfigService pipeline.PropertiesConfigService deploymentTemplateHistoryService history.DeploymentTemplateHistoryService environmentRepository repository3.EnvironmentRepository @@ -95,7 +97,7 @@ func NewDeploymentTemplateServiceImpl(Logger *zap.SugaredLogger, chartService ch helmAppService client.HelmAppService, chartRepository chartRepoRepository.ChartRepository, chartTemplateServiceImpl util.ChartTemplateService, - helmAppClient client.HelmAppClient, + helmAppClient gRPC.HelmAppClient, K8sUtil *k8s.K8sUtil, propertiesConfigService pipeline.PropertiesConfigService, deploymentTemplateHistoryService history.DeploymentTemplateHistoryService, @@ -340,18 +342,18 @@ func (impl DeploymentTemplateServiceImpl) GenerateManifest(ctx context.Context, impl.Logger.Errorw("exception caught in getting k8sServerVersion", "err", err) return nil, err } - installReleaseRequest := &client.InstallReleaseRequest{ + installReleaseRequest := &gRPC.InstallReleaseRequest{ ChartName: template, ChartVersion: version, ValuesYaml: valuesYaml, K8SVersion: k8sServerVersion.String(), ChartRepository: ChartRepository, ReleaseIdentifier: ReleaseIdentifier, - ChartContent: &client.ChartContent{ + ChartContent: &gRPC.ChartContent{ Content: chartBytes, }, } - config, err := impl.helmAppService.GetClusterConf(client.DEFAULT_CLUSTER_ID) + config, err := impl.helmAppService.GetClusterConf(bean.DEFAULT_CLUSTER_ID) if err != nil { impl.Logger.Errorw("error in fetching cluster detail", "clusterId", 1, "err", err) return nil, err diff --git a/pkg/generateManifest/DeployementTemplateService_test.go b/pkg/generateManifest/DeployementTemplateService_test.go index 3552db8fce..f90fa58a70 100644 --- a/pkg/generateManifest/DeployementTemplateService_test.go +++ b/pkg/generateManifest/DeployementTemplateService_test.go @@ -6,7 +6,7 @@ import ( client2 "github.com/devtron-labs/authenticator/client" "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/devtron/api/bean" - client "github.com/devtron-labs/devtron/api/helm-app" + client "github.com/devtron-labs/devtron/api/helm-app/gRPC" mocks4 "github.com/devtron-labs/devtron/api/helm-app/mocks" "github.com/devtron-labs/devtron/internal/sql/repository" mocks3 "github.com/devtron-labs/devtron/internal/sql/repository/mocks" diff --git a/pkg/k8s/K8sCommonService.go b/pkg/k8s/K8sCommonService.go index 6ff2385e6a..96a27ca3eb 100644 --- a/pkg/k8s/K8sCommonService.go +++ b/pkg/k8s/K8sCommonService.go @@ -7,7 +7,7 @@ import ( "github.com/devtron-labs/common-lib/utils/k8s" k8sCommonBean "github.com/devtron-labs/common-lib/utils/k8s/commonBean" "github.com/devtron-labs/devtron/api/bean" - "github.com/devtron-labs/devtron/api/helm-app" + "github.com/devtron-labs/devtron/api/helm-app/service" util2 "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/cluster" bean3 "github.com/devtron-labs/devtron/pkg/k8s/application/bean" @@ -163,7 +163,7 @@ func (impl *K8sCommonServiceImpl) FilterK8sResources(ctx context.Context, resour req := ResourceRequestBean{ AppId: appId, ClusterId: appDetail.ClusterId, - AppIdentifier: &client.AppIdentifier{ + AppIdentifier: &service.AppIdentifier{ ClusterId: appDetail.ClusterId, }, K8sRequest: &k8s.K8sRequestBean{ diff --git a/pkg/k8s/application/k8sApplicationService.go b/pkg/k8s/application/k8sApplicationService.go index 6186b99526..0f18a1f7a7 100644 --- a/pkg/k8s/application/k8sApplicationService.go +++ b/pkg/k8s/application/k8sApplicationService.go @@ -5,6 +5,8 @@ import ( "encoding/json" "errors" "fmt" + "github.com/devtron-labs/devtron/api/helm-app/gRPC" + client "github.com/devtron-labs/devtron/api/helm-app/service" "io" "net/http" "strconv" @@ -16,7 +18,6 @@ import ( k8sObjectUtils "github.com/devtron-labs/common-lib/utils/k8sObjectsUtil" yamlUtil "github.com/devtron-labs/common-lib/utils/yaml" "github.com/devtron-labs/devtron/api/connector" - client "github.com/devtron-labs/devtron/api/helm-app" "github.com/devtron-labs/devtron/api/helm-app/openapiClient" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" "github.com/devtron-labs/devtron/pkg/cluster" @@ -388,7 +389,7 @@ func (impl *K8sApplicationServiceImpl) ValidateResourceRequest(ctx context.Conte return impl.validateContainerNameIfReqd(valid, request, app), nil } -func (impl *K8sApplicationServiceImpl) validateContainerNameIfReqd(valid bool, request *k8s2.K8sRequestBean, app *client.AppDetail) bool { +func (impl *K8sApplicationServiceImpl) validateContainerNameIfReqd(valid bool, request *k8s2.K8sRequestBean, app *gRPC.AppDetail) bool { if !valid { requestContainerName := request.PodLogsRequest.ContainerName podName := request.ResourceIdentifier.Name diff --git a/pkg/k8s/application/k8sApplicationService_test.go b/pkg/k8s/application/k8sApplicationService_test.go index 12ce800ddf..6b68e2cbae 100644 --- a/pkg/k8s/application/k8sApplicationService_test.go +++ b/pkg/k8s/application/k8sApplicationService_test.go @@ -5,7 +5,7 @@ import ( "encoding/json" "fmt" k8s2 "github.com/devtron-labs/common-lib/utils/k8s" - client "github.com/devtron-labs/devtron/api/helm-app" + client "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/pkg/cluster" "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/k8s" diff --git a/pkg/k8s/application/mocks/K8sApplicationService.go b/pkg/k8s/application/mocks/K8sApplicationService.go index 638606d14d..434f01ffd4 100644 --- a/pkg/k8s/application/mocks/K8sApplicationService.go +++ b/pkg/k8s/application/mocks/K8sApplicationService.go @@ -5,7 +5,7 @@ package mocks import ( "github.com/devtron-labs/common-lib/utils/k8s" bean "github.com/devtron-labs/devtron/api/bean" - client "github.com/devtron-labs/devtron/api/helm-app" + client "github.com/devtron-labs/devtron/api/helm-app/service" k8s2 "github.com/devtron-labs/devtron/pkg/k8s" bean2 "github.com/devtron-labs/devtron/pkg/k8s/application/bean" diff --git a/pkg/k8s/bean.go b/pkg/k8s/bean.go index 23229ceaf4..d684e9076d 100644 --- a/pkg/k8s/bean.go +++ b/pkg/k8s/bean.go @@ -2,7 +2,7 @@ package k8s import ( "github.com/devtron-labs/common-lib/utils/k8s" - client "github.com/devtron-labs/devtron/api/helm-app" + client "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/pkg/k8s/application/bean" ) diff --git a/pkg/kubernetesResourceAuditLogs/kubernetesResourceHistoryService.go b/pkg/kubernetesResourceAuditLogs/kubernetesResourceHistoryService.go index 10ab04489d..859a5f258b 100644 --- a/pkg/kubernetesResourceAuditLogs/kubernetesResourceHistoryService.go +++ b/pkg/kubernetesResourceAuditLogs/kubernetesResourceHistoryService.go @@ -3,7 +3,7 @@ package kubernetesResourceAuditLogs import ( "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" "github.com/devtron-labs/common-lib/utils/k8s" - client "github.com/devtron-labs/devtron/api/helm-app" + client "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/internal/sql/repository/app" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/kubernetesResourceAuditLogs/repository" diff --git a/pkg/module/ModuleCronService.go b/pkg/module/ModuleCronService.go index 09e55347b2..03bca9094a 100644 --- a/pkg/module/ModuleCronService.go +++ b/pkg/module/ModuleCronService.go @@ -21,7 +21,8 @@ import ( "context" "encoding/json" "fmt" - client "github.com/devtron-labs/devtron/api/helm-app" + "github.com/devtron-labs/devtron/api/helm-app/gRPC" + client "github.com/devtron-labs/devtron/api/helm-app/service" moduleRepo "github.com/devtron-labs/devtron/pkg/module/repo" moduleDataStore "github.com/devtron-labs/devtron/pkg/module/store" serverBean "github.com/devtron-labs/devtron/pkg/server/bean" @@ -158,7 +159,7 @@ func (impl *ModuleCronServiceImpl) handleModuleStatus(moduleNameInput string) { } -func (impl *ModuleCronServiceImpl) saveModuleResourcesStatus(moduleId int, appDetail *client.AppDetail) error { +func (impl *ModuleCronServiceImpl) saveModuleResourcesStatus(moduleId int, appDetail *gRPC.AppDetail) error { impl.logger.Infow("updating module resources status", "moduleId", moduleId) if appDetail == nil || appDetail.ResourceTreeResponse == nil { return nil @@ -233,7 +234,7 @@ func (impl *ModuleCronServiceImpl) saveModuleResourcesStatus(moduleId int, appDe return nil } -func (impl *ModuleCronServiceImpl) buildResourceTreeFilter(moduleName string) (*client.ResourceTreeFilter, error) { +func (impl *ModuleCronServiceImpl) buildResourceTreeFilter(moduleName string) (*gRPC.ResourceTreeFilter, error) { moduleMetaData, err := impl.moduleServiceHelper.GetModuleMetadata(moduleName) if err != nil { impl.logger.Errorw("Error in getting module metadata", "moduleName", moduleName, "err", err) @@ -254,13 +255,13 @@ func (impl *ModuleCronServiceImpl) buildResourceTreeFilter(moduleName string) (* return nil, err } - var resourceTreeFilter *client.ResourceTreeFilter + var resourceTreeFilter *gRPC.ResourceTreeFilter // handle global filter globalFilter := resourceFilterIfaceValue.GlobalFilter if globalFilter != nil { - resourceTreeFilter = &client.ResourceTreeFilter{ - GlobalFilter: &client.ResourceIdentifier{ + resourceTreeFilter = &gRPC.ResourceTreeFilter{ + GlobalFilter: &gRPC.ResourceIdentifier{ Labels: globalFilter.Labels, }, } @@ -268,21 +269,21 @@ func (impl *ModuleCronServiceImpl) buildResourceTreeFilter(moduleName string) (* } // otherwise handle gvk level - var resourceFilters []*client.ResourceFilter + var resourceFilters []*gRPC.ResourceFilter for _, gvkLevelFilters := range resourceFilterIfaceValue.GvkLevelFilters { gvk := gvkLevelFilters.Gvk - resourceFilters = append(resourceFilters, &client.ResourceFilter{ - Gvk: &client.Gvk{ + resourceFilters = append(resourceFilters, &gRPC.ResourceFilter{ + Gvk: &gRPC.Gvk{ Group: gvk.Group, Version: gvk.Version, Kind: gvk.Kind, }, - ResourceIdentifier: &client.ResourceIdentifier{ + ResourceIdentifier: &gRPC.ResourceIdentifier{ Labels: gvkLevelFilters.ResourceIdentifier.Labels, }, }) } - resourceTreeFilter = &client.ResourceTreeFilter{ + resourceTreeFilter = &gRPC.ResourceTreeFilter{ ResourceFilters: resourceFilters, } return resourceTreeFilter, nil diff --git a/pkg/module/ModuleService.go b/pkg/module/ModuleService.go index 9e677148fc..297888cfea 100644 --- a/pkg/module/ModuleService.go +++ b/pkg/module/ModuleService.go @@ -21,7 +21,8 @@ import ( "context" "errors" "github.com/caarlos0/env/v6" - client "github.com/devtron-labs/devtron/api/helm-app" + "github.com/devtron-labs/devtron/api/helm-app/gRPC" + client "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/internal/sql/repository/security" moduleRepo "github.com/devtron-labs/devtron/pkg/module/repo" moduleUtil "github.com/devtron-labs/devtron/pkg/module/util" @@ -395,7 +396,7 @@ func (impl ModuleServiceImpl) HandleModuleAction(userId int32, moduleName string // HELM_OPERATION Starts devtronHelmAppIdentifier := impl.helmAppService.GetDevtronHelmAppIdentifier() - chartRepository := &client.ChartRepository{ + chartRepository := &gRPC.ChartRepository{ Name: impl.serverEnvConfig.DevtronHelmRepoName, Url: impl.serverEnvConfig.DevtronHelmRepoUrl, } diff --git a/pkg/pipeline/AppDeploymentTypeChangeManager.go b/pkg/pipeline/AppDeploymentTypeChangeManager.go index 5f340c755f..fcc6e1563c 100644 --- a/pkg/pipeline/AppDeploymentTypeChangeManager.go +++ b/pkg/pipeline/AppDeploymentTypeChangeManager.go @@ -21,7 +21,7 @@ import ( "context" "fmt" "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" - "github.com/devtron-labs/devtron/api/helm-app" + "github.com/devtron-labs/devtron/api/helm-app/service" application2 "github.com/devtron-labs/devtron/client/argocdServer/application" "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/appStatus" @@ -62,7 +62,7 @@ type AppDeploymentTypeChangeManagerImpl struct { workflowDagExecutor WorkflowDagExecutor appService app2.AppService appStatusRepository appStatus.AppStatusRepository - helmAppService client.HelmAppService + helmAppService service.HelmAppService application application2.ServiceClient appArtifactManager AppArtifactManager @@ -77,7 +77,7 @@ func NewAppDeploymentTypeChangeManagerImpl( workflowDagExecutor WorkflowDagExecutor, appService app2.AppService, appStatusRepository appStatus.AppStatusRepository, - helmAppService client.HelmAppService, + helmAppService service.HelmAppService, application application2.ServiceClient, appArtifactManager AppArtifactManager, cdPipelineConfigService CdPipelineConfigService, @@ -628,7 +628,7 @@ func (impl *AppDeploymentTypeChangeManagerImpl) FetchDeletedApp(ctx context.Cont deploymentAppName := fmt.Sprintf("%s-%s", pipeline.App.AppName, pipeline.Environment.Name) var err error if pipeline.DeploymentAppType == string(bean.ArgoCd) { - appIdentifier := &client.AppIdentifier{ + appIdentifier := &service.AppIdentifier{ ClusterId: pipeline.Environment.ClusterId, ReleaseName: pipeline.DeploymentAppName, Namespace: pipeline.Environment.Namespace, @@ -720,7 +720,7 @@ func (impl *AppDeploymentTypeChangeManagerImpl) deleteHelmApp(ctx context.Contex } // create app identifier - appIdentifier := &client.AppIdentifier{ + appIdentifier := &service.AppIdentifier{ ClusterId: pipeline.Environment.ClusterId, ReleaseName: pipeline.DeploymentAppName, Namespace: pipeline.Environment.Namespace, diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index 7737a2668d..df7c42b4ed 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -22,6 +22,7 @@ import ( "context" "errors" "fmt" + client "github.com/devtron-labs/devtron/api/helm-app/service" "os" "path/filepath" "strconv" @@ -34,7 +35,6 @@ import ( pubub "github.com/devtron-labs/common-lib/pubsub-lib" "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/devtron/api/bean" - client "github.com/devtron-labs/devtron/api/helm-app" "github.com/devtron-labs/devtron/client/argocdServer" "github.com/devtron-labs/devtron/client/argocdServer/application" client2 "github.com/devtron-labs/devtron/client/events" diff --git a/pkg/pipeline/DeploymentPipelineConfigService.go b/pkg/pipeline/DeploymentPipelineConfigService.go index 98e9c35512..3cef327795 100644 --- a/pkg/pipeline/DeploymentPipelineConfigService.go +++ b/pkg/pipeline/DeploymentPipelineConfigService.go @@ -24,8 +24,8 @@ import ( "fmt" application2 "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" bean2 "github.com/devtron-labs/devtron/api/bean" - client "github.com/devtron-labs/devtron/api/helm-app" models2 "github.com/devtron-labs/devtron/api/helm-app/models" + client "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/client/argocdServer" "github.com/devtron-labs/devtron/client/argocdServer/application" "github.com/devtron-labs/devtron/internal/sql/models" diff --git a/pkg/pipeline/DockerRegistryConfig.go b/pkg/pipeline/DockerRegistryConfig.go index 2bc7a6c941..e327c62d3c 100644 --- a/pkg/pipeline/DockerRegistryConfig.go +++ b/pkg/pipeline/DockerRegistryConfig.go @@ -20,7 +20,8 @@ package pipeline import ( "context" "fmt" - client "github.com/devtron-labs/devtron/api/helm-app" + bean2 "github.com/devtron-labs/devtron/api/helm-app/gRPC" + client "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/sql" "github.com/go-pg/pg" @@ -863,7 +864,7 @@ func (impl DockerRegistryConfigImpl) ValidateRegistryCredentials(bean *types.Doc bean.RegistryType == repository.REGISTRYTYPE_OTHER { return true } - request := &client.RegistryCredential{ + request := &bean2.RegistryCredential{ RegistryUrl: bean.RegistryURL, Username: bean.Username, Password: bean.Password, diff --git a/pkg/pipeline/DockerRegistryConfig_test.go b/pkg/pipeline/DockerRegistryConfig_test.go index 139b810cb9..2a5e20e107 100644 --- a/pkg/pipeline/DockerRegistryConfig_test.go +++ b/pkg/pipeline/DockerRegistryConfig_test.go @@ -3,7 +3,7 @@ package pipeline import ( "context" "fmt" - client "github.com/devtron-labs/devtron/api/helm-app" + client "github.com/devtron-labs/devtron/api/helm-app/gRPC" "github.com/devtron-labs/devtron/api/helm-app/mocks" repository "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" "github.com/devtron-labs/devtron/pkg/dockerRegistry" diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index f6fb6387ba..ffd655377c 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -22,6 +22,9 @@ import ( "encoding/json" errors3 "errors" "fmt" + bean5 "github.com/devtron-labs/devtron/api/helm-app/bean" + "github.com/devtron-labs/devtron/api/helm-app/gRPC" + client2 "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" @@ -39,7 +42,6 @@ import ( "github.com/devtron-labs/common-lib/pubsub-lib/model" util5 "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/common-lib/utils/k8s/health" - client2 "github.com/devtron-labs/devtron/api/helm-app" "github.com/devtron-labs/devtron/client/argocdServer" "github.com/devtron-labs/devtron/client/argocdServer/application" application2 "github.com/devtron-labs/devtron/client/argocdServer/application" @@ -180,17 +182,18 @@ type WorkflowDagExecutorImpl struct { configMapRepository chartConfig.ConfigMapRepository configMapHistoryRepository repository3.ConfigMapHistoryRepository helmAppService client2.HelmAppService - helmAppClient client2.HelmAppClient - environmentConfigRepository chartConfig.EnvConfigOverrideRepository - mergeUtil *util.MergeUtil - acdClient application2.ServiceClient - argoClientWrapperService argocdServer.ArgoClientWrapperService - customTagService CustomTagService - ACDConfig *argocdServer.ACDConfig - deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService - chartRefService chartRef.ChartRefService - gitOpsConfigReadService config.GitOpsConfigReadService - gitOpsRemoteOperationService remote.GitOpsRemoteOperationService + //TODO fix me next + helmAppClient gRPC.HelmAppClient //TODO refactoring: use helm app service instead + environmentConfigRepository chartConfig.EnvConfigOverrideRepository + mergeUtil *util.MergeUtil + acdClient application2.ServiceClient + argoClientWrapperService argocdServer.ArgoClientWrapperService + customTagService CustomTagService + ACDConfig *argocdServer.ACDConfig + deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService + chartRefService chartRef.ChartRefService + gitOpsConfigReadService config.GitOpsConfigReadService + gitOpsRemoteOperationService remote.GitOpsRemoteOperationService } const kedaAutoscaling = "kedaAutoscaling" @@ -281,7 +284,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi configMapRepository chartConfig.ConfigMapRepository, configMapHistoryRepository repository3.ConfigMapHistoryRepository, helmAppService client2.HelmAppService, - helmAppClient client2.HelmAppClient, + helmAppClient gRPC.HelmAppClient, environmentConfigRepository chartConfig.EnvConfigOverrideRepository, mergeUtil *util.MergeUtil, acdClient application2.ServiceClient, @@ -462,7 +465,7 @@ func (impl *WorkflowDagExecutorImpl) UpdateWorkflowRunnerStatusForDeployment(app if err != nil { impl.logger.Errorw("error in getting helm app release status", "appIdentifier", appIdentifier, "err", err) // Handle release not found errors - if skipReleaseNotFound && util.GetGRPCErrorDetailedMessage(err) != client2.ErrReleaseNotFound { + if skipReleaseNotFound && util.GetGRPCErrorDetailedMessage(err) != bean5.ErrReleaseNotFound { //skip this error and continue for next workflow status impl.logger.Warnw("found error, skipping helm apps status update for this trigger", "appIdentifier", appIdentifier, "err", err) return false @@ -3515,7 +3518,7 @@ func (impl *WorkflowDagExecutorImpl) createHelmAppForCdPipeline(overrideRequest releaseName := pipeline.DeploymentAppName cluster := envOverride.Environment.Cluster bearerToken := cluster.Config[util5.BearerToken] - clusterConfig := &client2.ClusterConfig{ + clusterConfig := &gRPC.ClusterConfig{ ClusterName: cluster.ClusterName, Token: bearerToken, ApiServerUrl: cluster.ServerUrl, @@ -3526,18 +3529,18 @@ func (impl *WorkflowDagExecutorImpl) createHelmAppForCdPipeline(overrideRequest clusterConfig.CertData = cluster.Config[util5.CertData] clusterConfig.CaData = cluster.Config[util5.CertificateAuthorityData] } - releaseIdentifier := &client2.ReleaseIdentifier{ + releaseIdentifier := &gRPC.ReleaseIdentifier{ ReleaseName: releaseName, ReleaseNamespace: envOverride.Namespace, ClusterConfig: clusterConfig, } if pipeline.DeploymentAppCreated { - req := &client2.UpgradeReleaseRequest{ + req := &gRPC.UpgradeReleaseRequest{ ReleaseIdentifier: releaseIdentifier, ValuesYaml: mergeAndSave, - HistoryMax: impl.helmAppService.GetRevisionHistoryMaxValue(client2.SOURCE_DEVTRON_APP), - ChartContent: &client2.ChartContent{Content: referenceChartByte}, + HistoryMax: impl.helmAppService.GetRevisionHistoryMaxValue(bean5.SOURCE_DEVTRON_APP), + ChartContent: &gRPC.ChartContent{Content: referenceChartByte}, } if impl.appService.IsDevtronAsyncInstallModeEnabled(bean2.Helm) { req.RunInCtx = true @@ -4245,11 +4248,11 @@ func (impl *WorkflowDagExecutorImpl) updatePipeline(pipeline *pipelineConfig.Pip } // helmInstallReleaseWithCustomChart performs helm install with custom chart -func (impl *WorkflowDagExecutorImpl) helmInstallReleaseWithCustomChart(ctx context.Context, releaseIdentifier *client2.ReleaseIdentifier, referenceChartByte []byte, valuesYaml string) (*client2.HelmInstallCustomResponse, error) { +func (impl *WorkflowDagExecutorImpl) helmInstallReleaseWithCustomChart(ctx context.Context, releaseIdentifier *gRPC.ReleaseIdentifier, referenceChartByte []byte, valuesYaml string) (*gRPC.HelmInstallCustomResponse, error) { - helmInstallRequest := client2.HelmInstallCustomRequest{ + helmInstallRequest := gRPC.HelmInstallCustomRequest{ ValuesYaml: valuesYaml, - ChartContent: &client2.ChartContent{Content: referenceChartByte}, + ChartContent: &gRPC.ChartContent{Content: referenceChartByte}, ReleaseIdentifier: releaseIdentifier, } if impl.appService.IsDevtronAsyncInstallModeEnabled(bean2.Helm) { diff --git a/pkg/server/ServerCacheService.go b/pkg/server/ServerCacheService.go index 8ac2b33449..27d67a85de 100644 --- a/pkg/server/ServerCacheService.go +++ b/pkg/server/ServerCacheService.go @@ -19,7 +19,7 @@ package server import ( "context" - client "github.com/devtron-labs/devtron/api/helm-app" + client "github.com/devtron-labs/devtron/api/helm-app/service" serverBean "github.com/devtron-labs/devtron/pkg/server/bean" serverEnvConfig "github.com/devtron-labs/devtron/pkg/server/config" serverDataStore "github.com/devtron-labs/devtron/pkg/server/store" diff --git a/pkg/server/ServerService.go b/pkg/server/ServerService.go index 731bce828a..75a19c99f6 100644 --- a/pkg/server/ServerService.go +++ b/pkg/server/ServerService.go @@ -20,7 +20,8 @@ package server import ( "context" "errors" - client "github.com/devtron-labs/devtron/api/helm-app" + "github.com/devtron-labs/devtron/api/helm-app/gRPC" + client "github.com/devtron-labs/devtron/api/helm-app/service" moduleRepo "github.com/devtron-labs/devtron/pkg/module/repo" moduleUtil "github.com/devtron-labs/devtron/pkg/module/util" serverBean "github.com/devtron-labs/devtron/pkg/server/bean" @@ -130,7 +131,7 @@ func (impl ServerServiceImpl) HandleServerAction(userId int32, serverActionReque // HELM_OPERATION Starts devtronHelmAppIdentifier := impl.helmAppService.GetDevtronHelmAppIdentifier() - chartRepository := &client.ChartRepository{ + chartRepository := &gRPC.ChartRepository{ Name: impl.serverEnvConfig.DevtronHelmRepoName, Url: impl.serverEnvConfig.DevtronHelmRepoUrl, } diff --git a/pkg/webhook/helm/WebhookHelmService.go b/pkg/webhook/helm/WebhookHelmService.go index f52f453cb0..1058bd728c 100644 --- a/pkg/webhook/helm/WebhookHelmService.go +++ b/pkg/webhook/helm/WebhookHelmService.go @@ -20,7 +20,9 @@ package webhookHelm import ( "context" "fmt" - client "github.com/devtron-labs/devtron/api/helm-app" + "github.com/devtron-labs/devtron/api/helm-app/bean" + bean2 "github.com/devtron-labs/devtron/api/helm-app/gRPC" + client "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/api/restHandler/common" "github.com/devtron-labs/devtron/pkg/attributes" "github.com/devtron-labs/devtron/pkg/chartRepo" @@ -119,15 +121,15 @@ func (impl WebhookHelmServiceImpl) CreateOrUpdateHelmApplication(ctx context.Con // STEP-6 install/update release chart := request.Chart chartRepo := request.Chart.Repo - installReleaseRequest := &client.InstallReleaseRequest{ - ReleaseIdentifier: &client.ReleaseIdentifier{ + installReleaseRequest := &bean2.InstallReleaseRequest{ + ReleaseIdentifier: &bean2.ReleaseIdentifier{ ReleaseName: appIdentifier.ReleaseName, ReleaseNamespace: appIdentifier.Namespace, }, ChartName: chart.ChartName, ChartVersion: chart.ChartVersion, ValuesYaml: request.ValuesOverrideYaml, - ChartRepository: &client.ChartRepository{ + ChartRepository: &bean2.ChartRepository{ Name: chartRepo.Name, Url: chartRepo.Identifier.Url, Username: chartRepo.Identifier.Username, @@ -135,9 +137,9 @@ func (impl WebhookHelmServiceImpl) CreateOrUpdateHelmApplication(ctx context.Con }, } if isInstalled { - updateReleaseRequest := &client.UpdateApplicationWithChartInfoRequestDto{ + updateReleaseRequest := &bean.UpdateApplicationWithChartInfoRequestDto{ InstallReleaseRequest: installReleaseRequest, - SourceAppType: client.SOURCE_HELM_APP, + SourceAppType: bean.SOURCE_HELM_APP, } res, err := impl.helmAppService.UpdateApplicationWithChartInfo(ctx, clusterId, updateReleaseRequest) if err != nil { diff --git a/wire_gen.go b/wire_gen.go index 990b564a79..4a2b07d4ff 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -28,6 +28,8 @@ import ( "github.com/devtron-labs/devtron/api/deployment" externalLink2 "github.com/devtron-labs/devtron/api/externalLink" client3 "github.com/devtron-labs/devtron/api/helm-app" + "github.com/devtron-labs/devtron/api/helm-app/gRPC" + "github.com/devtron-labs/devtron/api/helm-app/service" application3 "github.com/devtron-labs/devtron/api/k8s/application" capacity2 "github.com/devtron-labs/devtron/api/k8s/capacity" module2 "github.com/devtron-labs/devtron/api/module" @@ -78,14 +80,13 @@ import ( repository15 "github.com/devtron-labs/devtron/pkg/appStore/chartGroup/repository" "github.com/devtron-labs/devtron/pkg/appStore/chartProvider" "github.com/devtron-labs/devtron/pkg/appStore/deployment/common" - "github.com/devtron-labs/devtron/pkg/appStore/deployment/fullMode" repository3 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" - service2 "github.com/devtron-labs/devtron/pkg/appStore/deployment/service" + service3 "github.com/devtron-labs/devtron/pkg/appStore/deployment/service" "github.com/devtron-labs/devtron/pkg/appStore/deployment/tool" "github.com/devtron-labs/devtron/pkg/appStore/discover/repository" - service3 "github.com/devtron-labs/devtron/pkg/appStore/discover/service" + service4 "github.com/devtron-labs/devtron/pkg/appStore/discover/service" "github.com/devtron-labs/devtron/pkg/appStore/values/repository" - "github.com/devtron-labs/devtron/pkg/appStore/values/service" + service2 "github.com/devtron-labs/devtron/pkg/appStore/values/service" appWorkflow2 "github.com/devtron-labs/devtron/pkg/appWorkflow" "github.com/devtron-labs/devtron/pkg/attributes" "github.com/devtron-labs/devtron/pkg/auth/authentication" @@ -233,11 +234,11 @@ func InitializeApp() (*App, error) { userRepositoryImpl := repository4.NewUserRepositoryImpl(db, sugaredLogger) roleGroupRepositoryImpl := repository4.NewRoleGroupRepositoryImpl(db, sugaredLogger) clusterServiceImplExtended := cluster2.NewClusterServiceImplExtended(clusterRepositoryImpl, environmentRepositoryImpl, grafanaClientImpl, sugaredLogger, installedAppRepositoryImpl, k8sUtil, serviceClientImpl, k8sInformerFactoryImpl, gitOpsConfigRepositoryImpl, userAuthRepositoryImpl, userRepositoryImpl, roleGroupRepositoryImpl) - helmClientConfig, err := client3.GetConfig() + helmClientConfig, err := gRPC.GetConfig() if err != nil { return nil, err } - helmAppClientImpl := client3.NewHelmAppClientImpl(sugaredLogger, helmClientConfig) + helmAppClientImpl := gRPC.NewHelmAppClientImpl(sugaredLogger, helmClientConfig) pumpImpl := connector.NewPumpImpl(sugaredLogger) teamRepositoryImpl := team.NewTeamRepositoryImpl(db) appRepositoryImpl := app.NewAppRepositoryImpl(db, sugaredLogger) @@ -268,11 +269,11 @@ func InitializeApp() (*App, error) { userServiceImpl := user.NewUserServiceImpl(userAuthRepositoryImpl, sugaredLogger, userRepositoryImpl, roleGroupRepositoryImpl, sessionManager, userCommonServiceImpl, userAuditServiceImpl) userAuthServiceImpl := user.NewUserAuthServiceImpl(userAuthRepositoryImpl, sessionManager, loginService, sugaredLogger, userRepositoryImpl, roleGroupRepositoryImpl, userServiceImpl) environmentServiceImpl := cluster2.NewEnvironmentServiceImpl(environmentRepositoryImpl, clusterServiceImplExtended, sugaredLogger, k8sUtil, k8sInformerFactoryImpl, userAuthServiceImpl, attributesRepositoryImpl) - helmReleaseConfig, err := client3.GetHelmReleaseConfig() + helmReleaseConfig, err := service.GetHelmReleaseConfig() if err != nil { return nil, err } - helmAppServiceImpl := client3.NewHelmAppServiceImpl(sugaredLogger, clusterServiceImplExtended, helmAppClientImpl, pumpImpl, enforcerUtilHelmImpl, serverDataStoreServerDataStore, serverEnvConfigServerEnvConfig, appStoreApplicationVersionRepositoryImpl, environmentServiceImpl, pipelineRepositoryImpl, installedAppRepositoryImpl, appRepositoryImpl, clusterRepositoryImpl, k8sUtil, helmReleaseConfig) + helmAppServiceImpl := service.NewHelmAppServiceImpl(sugaredLogger, clusterServiceImplExtended, helmAppClientImpl, pumpImpl, enforcerUtilHelmImpl, serverDataStoreServerDataStore, serverEnvConfigServerEnvConfig, appStoreApplicationVersionRepositoryImpl, environmentServiceImpl, pipelineRepositoryImpl, installedAppRepositoryImpl, appRepositoryImpl, clusterRepositoryImpl, k8sUtil, helmReleaseConfig) serverCacheServiceImpl := server.NewServerCacheServiceImpl(sugaredLogger, serverEnvConfigServerEnvConfig, serverDataStoreServerDataStore, helmAppServiceImpl) moduleEnvConfig, err := module.ParseModuleEnvConfig() if err != nil { @@ -530,22 +531,21 @@ func InitializeApp() (*App, error) { pipelineStatusTimelineRestHandlerImpl := restHandler.NewPipelineStatusTimelineRestHandlerImpl(sugaredLogger, pipelineStatusTimelineServiceImpl, enforcerUtilImpl, enforcerImpl) pipelineConfigRouterImpl := router.NewPipelineRouterImpl(pipelineConfigRestHandlerImpl, appWorkflowRestHandlerImpl, webhookDataRestHandlerImpl, pipelineHistoryRestHandlerImpl, pipelineStatusTimelineRestHandlerImpl) appStoreVersionValuesRepositoryImpl := appStoreValuesRepository.NewAppStoreVersionValuesRepositoryImpl(sugaredLogger, db) - appStoreValuesServiceImpl := service.NewAppStoreValuesServiceImpl(sugaredLogger, appStoreApplicationVersionRepositoryImpl, installedAppRepositoryImpl, appStoreVersionValuesRepositoryImpl, userServiceImpl) + appStoreValuesServiceImpl := service2.NewAppStoreValuesServiceImpl(sugaredLogger, appStoreApplicationVersionRepositoryImpl, installedAppRepositoryImpl, appStoreVersionValuesRepositoryImpl, userServiceImpl) chartGroupDeploymentRepositoryImpl := repository15.NewChartGroupDeploymentRepositoryImpl(db, sugaredLogger) acdAuthConfig, err := util3.GetACDAuthConfig() if err != nil { return nil, err } - appStoreDeploymentCommonServiceImpl := appStoreDeploymentCommon.NewAppStoreDeploymentCommonServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, chartTemplateServiceImpl, gitFactory, gitOpsConfigReadServiceImpl, gitOpsRemoteOperationServiceImpl) - appStoreDeploymentFullModeServiceImpl := appStoreDeploymentFullMode.NewAppStoreDeploymentFullModeServiceImpl(sugaredLogger, applicationServiceClientImpl, argoK8sClientImpl, acdAuthConfig, argoUserServiceImpl, pipelineStatusTimelineServiceImpl, appStoreDeploymentCommonServiceImpl, argoClientWrapperServiceImpl, pubSubClientServiceImpl, installedAppVersionHistoryRepositoryImpl, acdConfig, gitOpsConfigReadServiceImpl, gitOpsRemoteOperationServiceImpl) clusterInstalledAppsRepositoryImpl := repository3.NewClusterInstalledAppsRepositoryImpl(db, sugaredLogger) - appStoreDeploymentHelmServiceImpl := appStoreDeploymentTool.NewAppStoreDeploymentHelmServiceImpl(sugaredLogger, helmAppServiceImpl, appStoreApplicationVersionRepositoryImpl, helmAppClientImpl, installedAppRepositoryImpl, appStoreDeploymentCommonServiceImpl, ociRegistryConfigRepositoryImpl, gitOpsRemoteOperationServiceImpl) - appStoreDeploymentArgoCdServiceImpl := appStoreDeploymentTool.NewAppStoreDeploymentArgoCdServiceImpl(sugaredLogger, appStoreDeploymentFullModeServiceImpl, applicationServiceClientImpl, chartGroupDeploymentRepositoryImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, appStoreDeploymentCommonServiceImpl, helmAppServiceImpl, appStatusServiceImpl, pipelineStatusTimelineServiceImpl, userServiceImpl, pipelineStatusTimelineRepositoryImpl, appStoreApplicationVersionRepositoryImpl, argoClientWrapperServiceImpl, acdConfig, gitOpsRemoteOperationServiceImpl) - serviceDeploymentServiceTypeConfig, err := service2.GetDeploymentServiceTypeConfig() + appStoreDeploymentHelmServiceImpl := appStoreDeploymentTool.NewAppStoreDeploymentHelmServiceImpl(sugaredLogger, helmAppServiceImpl, appStoreApplicationVersionRepositoryImpl, helmAppClientImpl, installedAppRepositoryImpl, ociRegistryConfigRepositoryImpl) + appStoreDeploymentCommonServiceImpl := appStoreDeploymentCommon.NewAppStoreDeploymentCommonServiceImpl(sugaredLogger, appStoreApplicationVersionRepositoryImpl, chartTemplateServiceImpl) + appStoreDeploymentArgoCdServiceImpl := appStoreDeploymentTool.NewAppStoreDeploymentArgoCdServiceImpl(sugaredLogger, applicationServiceClientImpl, argoK8sClientImpl, acdAuthConfig, chartGroupDeploymentRepositoryImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, appStoreDeploymentCommonServiceImpl, helmAppServiceImpl, appStatusServiceImpl, pipelineStatusTimelineServiceImpl, userServiceImpl, pipelineStatusTimelineRepositoryImpl, appStoreApplicationVersionRepositoryImpl, argoClientWrapperServiceImpl, acdConfig, gitOpsRemoteOperationServiceImpl, gitOpsConfigReadServiceImpl, environmentRepositoryImpl) + serviceDeploymentServiceTypeConfig, err := service3.GetDeploymentServiceTypeConfig() if err != nil { return nil, err } - appStoreDeploymentServiceImpl := service2.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, clusterInstalledAppsRepositoryImpl, appRepositoryImpl, appStoreDeploymentHelmServiceImpl, appStoreDeploymentArgoCdServiceImpl, environmentServiceImpl, clusterServiceImplExtended, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, installedAppVersionHistoryRepositoryImpl, gitOpsConfigRepositoryImpl, serviceDeploymentServiceTypeConfig, acdConfig, gitOpsConfigReadServiceImpl, gitOpsRemoteOperationServiceImpl) + appStoreDeploymentServiceImpl := service3.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, clusterInstalledAppsRepositoryImpl, appRepositoryImpl, appStoreDeploymentHelmServiceImpl, appStoreDeploymentArgoCdServiceImpl, environmentServiceImpl, clusterServiceImplExtended, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, installedAppVersionHistoryRepositoryImpl, gitOpsConfigRepositoryImpl, serviceDeploymentServiceTypeConfig, acdConfig, gitOpsConfigReadServiceImpl, gitOpsRemoteOperationServiceImpl) k8sResourceHistoryRepositoryImpl := repository16.NewK8sResourceHistoryRepositoryImpl(db, sugaredLogger) k8sResourceHistoryServiceImpl := kubernetesResourceAuditLogs.Newk8sResourceHistoryServiceImpl(k8sResourceHistoryRepositoryImpl, sugaredLogger, appRepositoryImpl, environmentRepositoryImpl) ephemeralContainersRepositoryImpl := repository2.NewEphemeralContainersRepositoryImpl(db) @@ -555,7 +555,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - installedAppServiceImpl, err := service2.NewInstalledAppServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, teamRepositoryImpl, appRepositoryImpl, applicationServiceClientImpl, appStoreValuesServiceImpl, pubSubClientServiceImpl, chartGroupDeploymentRepositoryImpl, environmentServiceImpl, acdAuthConfig, gitOpsConfigRepositoryImpl, userServiceImpl, appStoreDeploymentFullModeServiceImpl, appStoreDeploymentServiceImpl, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, helmAppClientImpl, helmAppServiceImpl, appStatusServiceImpl, k8sUtil, pipelineStatusTimelineServiceImpl, appStoreDeploymentCommonServiceImpl, k8sCommonServiceImpl, k8sApplicationServiceImpl, acdConfig, gitOpsConfigReadServiceImpl, gitOpsRemoteOperationServiceImpl) + installedAppServiceImpl, err := service3.NewInstalledAppServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, teamRepositoryImpl, appRepositoryImpl, applicationServiceClientImpl, appStoreValuesServiceImpl, pubSubClientServiceImpl, chartGroupDeploymentRepositoryImpl, environmentServiceImpl, userServiceImpl, acdAuthConfig, appStoreDeploymentServiceImpl, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, helmAppClientImpl, helmAppServiceImpl, appStatusServiceImpl, k8sUtil, pipelineStatusTimelineServiceImpl, k8sCommonServiceImpl, k8sApplicationServiceImpl, acdConfig, gitOpsRemoteOperationServiceImpl, appStoreDeploymentArgoCdServiceImpl) if err != nil { return nil, err } @@ -642,7 +642,7 @@ func InitializeApp() (*App, error) { installedAppRestHandlerImpl := appStore.NewInstalledAppRestHandlerImpl(sugaredLogger, userServiceImpl, enforcerImpl, enforcerUtilImpl, enforcerUtilHelmImpl, installedAppServiceImpl, validate, clusterServiceImplExtended, applicationServiceClientImpl, appStoreDeploymentServiceImpl, helmAppClientImpl, argoUserServiceImpl, cdApplicationStatusUpdateHandlerImpl, installedAppRepositoryImpl, appCrudOperationServiceImpl) appStoreValuesRestHandlerImpl := appStoreValues.NewAppStoreValuesRestHandlerImpl(sugaredLogger, userServiceImpl, appStoreValuesServiceImpl) appStoreValuesRouterImpl := appStoreValues.NewAppStoreValuesRouterImpl(appStoreValuesRestHandlerImpl) - appStoreServiceImpl := service3.NewAppStoreServiceImpl(sugaredLogger, appStoreApplicationVersionRepositoryImpl) + appStoreServiceImpl := service4.NewAppStoreServiceImpl(sugaredLogger, appStoreApplicationVersionRepositoryImpl) appStoreRestHandlerImpl := appStoreDiscover.NewAppStoreRestHandlerImpl(sugaredLogger, userServiceImpl, appStoreServiceImpl, enforcerImpl) appStoreDiscoverRouterImpl := appStoreDiscover.NewAppStoreDiscoverRouterImpl(appStoreRestHandlerImpl) chartProviderRestHandlerImpl := chartProvider2.NewChartProviderRestHandlerImpl(sugaredLogger, userServiceImpl, validate, chartProviderServiceImpl, enforcerImpl) @@ -733,7 +733,7 @@ func InitializeApp() (*App, error) { appRouterImpl := router.NewAppRouterImpl(sugaredLogger, appRestHandlerImpl) coreAppRestHandlerImpl := restHandler.NewCoreAppRestHandlerImpl(sugaredLogger, userServiceImpl, validate, enforcerUtilImpl, enforcerImpl, appCrudOperationServiceImpl, pipelineBuilderImpl, gitRegistryConfigImpl, chartServiceImpl, configMapServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, appWorkflowServiceImpl, materialRepositoryImpl, gitProviderRepositoryImpl, appWorkflowRepositoryImpl, environmentRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, teamServiceImpl, argoUserServiceImpl, pipelineStageServiceImpl, ciPipelineRepositoryImpl) coreAppRouterImpl := router.NewCoreAppRouterImpl(coreAppRestHandlerImpl) - helmAppRestHandlerImpl := client3.NewHelmAppRestHandlerImpl(sugaredLogger, helmAppServiceImpl, enforcerImpl, clusterServiceImplExtended, enforcerUtilHelmImpl, appStoreDeploymentCommonServiceImpl, userServiceImpl, attributesServiceImpl, serverEnvConfigServerEnvConfig) + helmAppRestHandlerImpl := client3.NewHelmAppRestHandlerImpl(sugaredLogger, helmAppServiceImpl, enforcerImpl, clusterServiceImplExtended, enforcerUtilHelmImpl, appStoreDeploymentServiceImpl, userServiceImpl, attributesServiceImpl, serverEnvConfigServerEnvConfig) helmAppRouterImpl := client3.NewHelmAppRouterImpl(helmAppRestHandlerImpl) k8sApplicationRestHandlerImpl := application3.NewK8sApplicationRestHandlerImpl(sugaredLogger, k8sApplicationServiceImpl, pumpImpl, terminalSessionHandlerImpl, enforcerImpl, enforcerUtilHelmImpl, enforcerUtilImpl, helmAppServiceImpl, userServiceImpl, k8sCommonServiceImpl, validate) k8sApplicationRouterImpl := application3.NewK8sApplicationRouterImpl(k8sApplicationRestHandlerImpl) @@ -743,7 +743,7 @@ func InitializeApp() (*App, error) { deploymentConfigRouterImpl := deployment.NewDeploymentRouterImpl(deploymentConfigRestHandlerImpl) dashboardTelemetryRestHandlerImpl := dashboardEvent.NewDashboardTelemetryRestHandlerImpl(sugaredLogger, telemetryEventClientImplExtended) dashboardTelemetryRouterImpl := dashboardEvent.NewDashboardTelemetryRouterImpl(dashboardTelemetryRestHandlerImpl) - commonDeploymentRestHandlerImpl := appStoreDeployment.NewCommonDeploymentRestHandlerImpl(sugaredLogger, userServiceImpl, enforcerImpl, enforcerUtilImpl, enforcerUtilHelmImpl, appStoreDeploymentServiceImpl, validate, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, helmAppRestHandlerImpl, argoUserServiceImpl) + commonDeploymentRestHandlerImpl := appStoreDeployment.NewCommonDeploymentRestHandlerImpl(sugaredLogger, userServiceImpl, enforcerImpl, enforcerUtilImpl, enforcerUtilHelmImpl, appStoreDeploymentServiceImpl, validate, helmAppServiceImpl, helmAppRestHandlerImpl, argoUserServiceImpl) commonDeploymentRouterImpl := appStoreDeployment.NewCommonDeploymentRouterImpl(commonDeploymentRestHandlerImpl) externalLinkMonitoringToolRepositoryImpl := externalLink.NewExternalLinkMonitoringToolRepositoryImpl(db) externalLinkIdentifierMappingRepositoryImpl := externalLink.NewExternalLinkIdentifierMappingRepositoryImpl(db) From 869c6e4869afebc3603142ea3c11b8a7716747a6 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 23 Jan 2024 11:42:05 +0530 Subject: [PATCH 30/64] minor cleanup --- pkg/chart/ChartService.go | 4 ---- pkg/pipeline/PropertiesConfig.go | 13 ------------- 2 files changed, 17 deletions(-) diff --git a/pkg/chart/ChartService.go b/pkg/chart/ChartService.go index 58222ac4e2..efecaa0ac6 100644 --- a/pkg/chart/ChartService.go +++ b/pkg/chart/ChartService.go @@ -145,10 +145,6 @@ func (impl ChartServiceImpl) PatchEnvOverrides(values json.RawMessage, oldChartT return PatchWinterSoldierConfig(values, newChartType) } -type AppMetricsEnabled struct { - AppMetrics bool `json:"app-metrics"` -} - func (impl ChartServiceImpl) Create(templateRequest TemplateRequest, ctx context.Context) (*TemplateRequest, error) { err := impl.chartRefService.CheckChartExists(templateRequest.ChartRefId) if err != nil { diff --git a/pkg/pipeline/PropertiesConfig.go b/pkg/pipeline/PropertiesConfig.go index c1620cdd09..40adfb2f7b 100644 --- a/pkg/pipeline/PropertiesConfig.go +++ b/pkg/pipeline/PropertiesConfig.go @@ -28,7 +28,6 @@ import ( repository5 "github.com/devtron-labs/devtron/pkg/variables/repository" "time" - chartService "github.com/devtron-labs/devtron/pkg/chart" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/pipeline/history" @@ -331,18 +330,6 @@ func (impl PropertiesConfigServiceImpl) UpdateEnvironmentProperties(appId int, p return propertiesRequest, err } -func (impl PropertiesConfigServiceImpl) buildAppMetricsJson() ([]byte, error) { - appMetricsEnabled := chartService.AppMetricsEnabled{ - AppMetrics: true, - } - appMetricsJson, err := json.Marshal(appMetricsEnabled) - if err != nil { - impl.logger.Error(err) - return nil, err - } - return appMetricsJson, nil -} - func (impl PropertiesConfigServiceImpl) CreateIfRequired(chart *chartRepoRepository.Chart, environmentId int, userId int32, manualReviewed bool, chartStatus models.ChartStatus, isOverride, isAppMetricsEnabled bool, namespace string, IsBasicViewLocked bool, CurrentViewEditor models.ChartsViewEditorType, tx *pg.Tx) (*chartConfig.EnvConfigOverride, bool, error) { env, err := impl.environmentRepository.FindById(environmentId) if err != nil { From d1116982f4c941d72a8fb59fdbf396b37b4d273b Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 23 Jan 2024 12:17:14 +0530 Subject: [PATCH 31/64] renamed remote package to git --- cmd/external-app/wire_gen.go | 4 ++-- internal/util/GitCliUtil.go | 2 +- pkg/app/AppService.go | 6 +++--- pkg/app/ManifestPushService.go | 6 +++--- .../common/AppStoreDeploymentCommonService.go | 6 +++--- .../fullMode/AppStoreDeploymentFullModeService.go | 6 +++--- .../deployment/service/AppStoreDeploymentService.go | 6 +++--- .../service/AppStoreDeploymentService_test.go | 4 ++-- .../deployment/service/InstalledAppService.go | 8 ++++---- .../tool/AppStoreDeploymentArgoCdService.go | 6 +++--- .../deployment/tool/AppStoreDeploymentHelmService.go | 6 +++--- .../{remote => git}/GitOpsRemoteOperationService.go | 4 ++-- pkg/deployment/gitOps/{remote => git}/bean/bean.go | 0 pkg/deployment/gitOps/wire_gitOps.go | 6 +++--- pkg/pipeline/DeploymentPipelineConfigService.go | 8 ++++---- pkg/pipeline/PipelineStageServiceIT_test.go | 12 ++++++------ pkg/pipeline/WorkflowDagExecutor.go | 8 ++++---- .../deployment-chart_1-0-0/app-values.yaml | 4 ++-- .../deployment-chart_1-1-0/app-values.yaml | 4 ++-- .../deployment-chart_4-18-0/app-values.yaml | 4 ++-- .../deployment-chart_4-19-0/app-values.yaml | 4 ++-- .../reference-chart_3-12-0/app-values.yaml | 4 ++-- .../reference-chart_3-13-0/app-values.yaml | 4 ++-- .../reference-chart_4-11-0/app-values.yaml | 4 ++-- .../reference-chart_4-12-0/app-values.yaml | 4 ++-- .../reference-chart_4-13-0/app-values.yaml | 4 ++-- .../reference-chart_4-14-0/app-values.yaml | 4 ++-- .../reference-chart_4-15-0/app-values.yaml | 4 ++-- .../reference-chart_4-16-0/app-values.yaml | 4 ++-- .../reference-chart_4-17-0/app-values.yaml | 4 ++-- .../reference-chart_4-18-0/app-values.yaml | 4 ++-- .../reference-chart_5-0-0/app-values.yaml | 4 ++-- .../statefulset-chart_4-18-0/app-values.yaml | 4 ++-- .../statefulset-chart_4-19-0/app-values.yaml | 4 ++-- .../statefulset-chart_5-0-0/app-values.yaml | 4 ++-- scripts/sql/49_plugin_integration.up.sql | 6 +++--- util/RequestUtil.go | 2 +- wire_gen.go | 10 +++++----- 38 files changed, 94 insertions(+), 94 deletions(-) rename pkg/deployment/gitOps/{remote => git}/GitOpsRemoteOperationService.go (99%) rename pkg/deployment/gitOps/{remote => git}/bean/bean.go (100%) diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index 0ba02067c4..b41b6fff67 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -69,7 +69,7 @@ import ( "github.com/devtron-labs/devtron/pkg/clusterTerminalAccess" delete2 "github.com/devtron-labs/devtron/pkg/delete" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" - "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" "github.com/devtron-labs/devtron/pkg/externalLink" "github.com/devtron-labs/devtron/pkg/genericNotes" repository6 "github.com/devtron-labs/devtron/pkg/genericNotes/repository" @@ -244,7 +244,7 @@ func InitializeApp() (*App, error) { return nil, err } gitOpsConfigReadServiceImpl := config.NewGitOpsConfigReadServiceImpl(sugaredLogger, gitOpsConfigRepositoryImpl, userServiceImpl, globalEnvVariables) - gitOpsRemoteOperationServiceImpl := remote.NewGitOpsRemoteOperationServiceImpl(sugaredLogger, gitFactory, gitOpsConfigReadServiceImpl, chartTemplateServiceImpl) + gitOpsRemoteOperationServiceImpl := git.NewGitOpsRemoteOperationServiceImpl(sugaredLogger, gitFactory, gitOpsConfigReadServiceImpl, chartTemplateServiceImpl) appStoreDeploymentCommonServiceImpl := appStoreDeploymentCommon.NewAppStoreDeploymentCommonServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, chartTemplateServiceImpl, gitFactory, gitOpsConfigReadServiceImpl, gitOpsRemoteOperationServiceImpl) attributesServiceImpl := attributes.NewAttributesServiceImpl(sugaredLogger, attributesRepositoryImpl) helmAppRestHandlerImpl := client2.NewHelmAppRestHandlerImpl(sugaredLogger, helmAppServiceImpl, enforcerImpl, clusterServiceImpl, enforcerUtilHelmImpl, appStoreDeploymentCommonServiceImpl, userServiceImpl, attributesServiceImpl, serverEnvConfigServerEnvConfig) diff --git a/internal/util/GitCliUtil.go b/internal/util/GitCliUtil.go index 3996751b6b..54d9281a2a 100644 --- a/internal/util/GitCliUtil.go +++ b/internal/util/GitCliUtil.go @@ -137,7 +137,7 @@ func (impl *GitCliUtil) Clone(rootDir string, remoteUrl string, username string, } response, errMsg, err = impl.Fetch(rootDir, username, password) if err == nil && errMsg == "" { - impl.logger.Warn("git fetch completed, pulling master branch data from remote origin") + impl.logger.Warn("git fetch completed, pulling master branch data from git origin") response, errMsg, err = impl.ListBranch(rootDir, username, password) if err != nil { impl.logger.Errorw("error on git pull", "response", response, "errMsg", errMsg, "err", err) diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index 94e74f31c4..173f503083 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -23,7 +23,7 @@ import ( "fmt" commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" - "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" bean3 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/bean" "io/ioutil" @@ -122,7 +122,7 @@ type AppServiceImpl struct { acdConfig *argocdServer.ACDConfig chartRefService chartRef.ChartRefService gitOpsConfigReadService config.GitOpsConfigReadService - gitOpsRemoteOperationService remote.GitOpsRemoteOperationService + gitOpsRemoteOperationService git.GitOpsRemoteOperationService } type AppService interface { @@ -177,7 +177,7 @@ func NewAppService( scopedVariableManager variables.ScopedVariableCMCSManager, acdConfig *argocdServer.ACDConfig, chartRefService chartRef.ChartRefService, gitOpsConfigReadService config.GitOpsConfigReadService, - gitOpsRemoteOperationService remote.GitOpsRemoteOperationService) *AppServiceImpl { + gitOpsRemoteOperationService git.GitOpsRemoteOperationService) *AppServiceImpl { appServiceImpl := &AppServiceImpl{ environmentConfigRepository: environmentConfigRepository, mergeUtil: mergeUtil, diff --git a/pkg/app/ManifestPushService.go b/pkg/app/ManifestPushService.go index 937023f39b..3e407dd935 100644 --- a/pkg/app/ManifestPushService.go +++ b/pkg/app/ManifestPushService.go @@ -9,7 +9,7 @@ import ( "github.com/devtron-labs/devtron/pkg/app/bean" status2 "github.com/devtron-labs/devtron/pkg/app/status" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" - "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" "go.opentelemetry.io/otel" "go.uber.org/zap" @@ -31,7 +31,7 @@ type GitOpsManifestPushServiceImpl struct { acdConfig *argocdServer.ACDConfig chartRefService chartRef.ChartRefService gitOpsConfigReadService config.GitOpsConfigReadService - gitOpsRemoteOperationService remote.GitOpsRemoteOperationService + gitOpsRemoteOperationService git.GitOpsRemoteOperationService } func NewGitOpsManifestPushServiceImpl(logger *zap.SugaredLogger, @@ -39,7 +39,7 @@ func NewGitOpsManifestPushServiceImpl(logger *zap.SugaredLogger, pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository, acdConfig *argocdServer.ACDConfig, chartRefService chartRef.ChartRefService, gitOpsConfigReadService config.GitOpsConfigReadService, - gitOpsRemoteOperationService remote.GitOpsRemoteOperationService) *GitOpsManifestPushServiceImpl { + gitOpsRemoteOperationService git.GitOpsRemoteOperationService) *GitOpsManifestPushServiceImpl { return &GitOpsManifestPushServiceImpl{ logger: logger, pipelineStatusTimelineService: pipelineStatusTimelineService, diff --git a/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go b/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go index 08460991af..7cbcb99d44 100644 --- a/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go +++ b/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go @@ -29,7 +29,7 @@ import ( repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" - "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" "github.com/go-pg/pg" "github.com/google/go-github/github" "github.com/microsoft/azure-devops-go-api/azuredevops" @@ -69,7 +69,7 @@ type AppStoreDeploymentCommonServiceImpl struct { chartTemplateService util.ChartTemplateService gitFactory *util.GitFactory gitOpsConfigReadService config.GitOpsConfigReadService - gitOpsRemoteOperationService remote.GitOpsRemoteOperationService + gitOpsRemoteOperationService git.GitOpsRemoteOperationService } func NewAppStoreDeploymentCommonServiceImpl( @@ -80,7 +80,7 @@ func NewAppStoreDeploymentCommonServiceImpl( chartTemplateService util.ChartTemplateService, gitFactory *util.GitFactory, gitOpsConfigReadService config.GitOpsConfigReadService, - gitOpsRemoteOperationService remote.GitOpsRemoteOperationService) *AppStoreDeploymentCommonServiceImpl { + gitOpsRemoteOperationService git.GitOpsRemoteOperationService) *AppStoreDeploymentCommonServiceImpl { return &AppStoreDeploymentCommonServiceImpl{ logger: logger, installedAppRepository: installedAppRepository, diff --git a/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go b/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go index cfacb79ca7..42f241b82a 100644 --- a/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go +++ b/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go @@ -26,7 +26,7 @@ import ( "github.com/devtron-labs/common-lib/pubsub-lib/model" commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" - "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" "time" "github.com/devtron-labs/devtron/client/argocdServer" @@ -73,7 +73,7 @@ type AppStoreDeploymentFullModeServiceImpl struct { installedAppRepositoryHistory repository4.InstalledAppVersionHistoryRepository ACDConfig *argocdServer.ACDConfig gitOpsConfigReadService config.GitOpsConfigReadService - gitOpsRemoteOperationService remote.GitOpsRemoteOperationService + gitOpsRemoteOperationService git.GitOpsRemoteOperationService } func NewAppStoreDeploymentFullModeServiceImpl(logger *zap.SugaredLogger, @@ -86,7 +86,7 @@ func NewAppStoreDeploymentFullModeServiceImpl(logger *zap.SugaredLogger, installedAppRepositoryHistory repository4.InstalledAppVersionHistoryRepository, ACDConfig *argocdServer.ACDConfig, gitOpsConfigReadService config.GitOpsConfigReadService, - gitOpsRemoteOperationService remote.GitOpsRemoteOperationService) *AppStoreDeploymentFullModeServiceImpl { + gitOpsRemoteOperationService git.GitOpsRemoteOperationService) *AppStoreDeploymentFullModeServiceImpl { appStoreDeploymentFullModeServiceImpl := &AppStoreDeploymentFullModeServiceImpl{ logger: logger, acdClient: acdClient, diff --git a/pkg/appStore/deployment/service/AppStoreDeploymentService.go b/pkg/appStore/deployment/service/AppStoreDeploymentService.go index c3221a7651..cc40832ae0 100644 --- a/pkg/appStore/deployment/service/AppStoreDeploymentService.go +++ b/pkg/appStore/deployment/service/AppStoreDeploymentService.go @@ -44,7 +44,7 @@ import ( cluster2 "github.com/devtron-labs/devtron/pkg/cluster" clusterRepository "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" - "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" "github.com/devtron-labs/devtron/pkg/sql" util2 "github.com/devtron-labs/devtron/util" "github.com/go-pg/pg" @@ -108,7 +108,7 @@ type AppStoreDeploymentServiceImpl struct { deploymentTypeConfig *DeploymentServiceTypeConfig aCDConfig *argocdServer.ACDConfig gitOpsConfigReadService config.GitOpsConfigReadService - gitOpsRemoteOperationService remote.GitOpsRemoteOperationService + gitOpsRemoteOperationService git.GitOpsRemoteOperationService } func NewAppStoreDeploymentServiceImpl(logger *zap.SugaredLogger, installedAppRepository repository.InstalledAppRepository, @@ -120,7 +120,7 @@ func NewAppStoreDeploymentServiceImpl(logger *zap.SugaredLogger, installedAppRep installedAppRepositoryHistory repository.InstalledAppVersionHistoryRepository, gitOpsRepository repository2.GitOpsConfigRepository, deploymentTypeConfig *DeploymentServiceTypeConfig, aCDConfig *argocdServer.ACDConfig, gitOpsConfigReadService config.GitOpsConfigReadService, - gitOpsRemoteOperationService remote.GitOpsRemoteOperationService) *AppStoreDeploymentServiceImpl { + gitOpsRemoteOperationService git.GitOpsRemoteOperationService) *AppStoreDeploymentServiceImpl { appStoreDeploymentServiceImpl := &AppStoreDeploymentServiceImpl{ logger: logger, diff --git a/pkg/appStore/deployment/service/AppStoreDeploymentService_test.go b/pkg/appStore/deployment/service/AppStoreDeploymentService_test.go index 79a6fd1f9f..01aabf55f1 100644 --- a/pkg/appStore/deployment/service/AppStoreDeploymentService_test.go +++ b/pkg/appStore/deployment/service/AppStoreDeploymentService_test.go @@ -43,7 +43,7 @@ func TestAppStoreDeploymentService(t *testing.T) { InstalledAppId: 0, InstalledAppVersionId: 0, AppStoreVersion: 6304, - ValuesOverrideYaml: "## @section Global parameters\n## Global Docker image parameters\n## Please, note that this will override the image parameters, including dependencies, configured to use the global value\n## Current available global Docker image parameters: imageRegistry, imagePullSecrets and storageClass\n\n## @param global.imageRegistry Global Docker image registry\n## @param global.imagePullSecrets Global Docker registry secret names as an array\n## @param global.storageClass Global StorageClass for Persistent Volume(s)\n##\nglobal:\n imageRegistry: \"\"\n ## E.g.\n ## imagePullSecrets:\n ## - myRegistryKeySecretName\n ##\n imagePullSecrets: []\n storageClass: \"\"\n\n## @section Common parameters\n\n## @param kubeVersion Override Kubernetes version\n##\nkubeVersion: \"\"\n## @param nameOverride String to partially override common.names.fullname template (will maintain the release name)\n##\nnameOverride: \"\"\n## @param fullnameOverride String to fully override common.names.fullname template\n##\nfullnameOverride: \"\"\n## @param clusterDomain Kubernetes Cluster Domain\n##\nclusterDomain: cluster.local\n## @param extraDeploy Extra objects to deploy (evaluated as a template)\n##\nextraDeploy: []\n## @param commonLabels Add labels to all the deployed resources\n##\ncommonLabels: {}\n## @param commonAnnotations Add annotations to all the deployed resources\n##\ncommonAnnotations: {}\n## Enable diagnostic mode in the deployment(s)/statefulset(s)\n##\ndiagnosticMode:\n ## @param diagnosticMode.enabled Enable diagnostic mode (all probes will be disabled and the command will be overridden)\n ##\n enabled: false\n ## @param diagnosticMode.command Command to override all containers in the the deployment(s)/statefulset(s)\n ##\n command:\n - sleep\n ## @param diagnosticMode.args Args to override all containers in the the deployment(s)/statefulset(s)\n ##\n args:\n - infinity\n\n## @section Airflow common parameters\n\n## Authentication parameters\n## ref: https://github.com/bitnami/containers/tree/main/bitnami/airflow#environment-variables\n##\nauth:\n ## @param auth.username Username to access web UI\n ##\n username: user\n ## @param auth.password Password to access web UI\n ##\n password: \"\"\n ## @param auth.fernetKey Fernet key to secure connections\n ## ref: https://airflow.readthedocs.io/en/stable/howto/secure-connections.html\n ## ref: https://bcb.github.io/airflow/fernet-key\n ##\n fernetKey: \"\"\n ## @param auth.secretKey Secret key to run your flask app\n ## ref: https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.html#secret-key\n ##\n secretKey: \"\"\n ## @param auth.existingSecret Name of an existing secret to use for Airflow credentials\n ## `auth.password`, `auth.fernetKey`, and `auth.secretKey` will be ignored and picked up from this secret\n ## The secret must contain the keys `airflow-password`, `airflow-fernet-key` and `airflow-secret-key'\n ## The value is evaluated as a template\n ##\n existingSecret: \"\"\n## @param executor Airflow executor. Allowed values: `SequentialExecutor`, `LocalExecutor`, `CeleryExecutor`, `KubernetesExecutor`, `CeleryKubernetesExecutor` and `LocalKubernetesExecutor`\n## ref: http://airflow.apache.org/docs/stable/executor/index.html\n##\nexecutor: CeleryExecutor\n## @param loadExamples Switch to load some Airflow examples\n##\nloadExamples: false\n## @param configuration Specify content for Airflow config file (auto-generated based on other env. vars otherwise)\n## e.g:\n## configuration: |-\n## [core]\n## dags_folder=/opt/bitnami/airflow/dags\n## ...\n##\nconfiguration: \"\"\n## @param existingConfigmap Name of an existing ConfigMap with the Airflow config file\n##\nexistingConfigmap: \"\"\n## Load custom DAGs from a ConfigMap\n## Note: an init container will be used to prepare the DAGs available in the ConfigMap to be consumed by Airflow containers\n##\ndags:\n ## @param dags.existingConfigmap Name of an existing ConfigMap with all the DAGs files you want to load in Airflow\n ##\n existingConfigmap: \"\"\n ## Bitnami Shell image\n ## ref: https://hub.docker.com/r/bitnami/bitnami-shell/tags/\n ## @param dags.image.registry Init container load-dags image registry\n ## @param dags.image.repository Init container load-dags image repository\n ## @param dags.image.tag Init container load-dags image tag (immutable tags are recommended)\n ## @param dags.image.digest Init container load-dags image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param dags.image.pullPolicy Init container load-dags image pull policy\n ## @param dags.image.pullSecrets Init container load-dags image pull secrets\n ##\n image:\n registry: docker.io\n repository: bitnami/bitnami-shell\n tag: 11-debian-11-r50\n digest: \"\"\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n## @param extraEnvVars Add extra environment variables for all the Airflow pods\n##\nextraEnvVars: []\n## @param extraEnvVarsCM ConfigMap with extra environment variables for all the Airflow pods\n##\nextraEnvVarsCM: \"\"\n## @param extraEnvVarsSecret Secret with extra environment variables for all the Airflow pods\n##\nextraEnvVarsSecret: \"\"\n## @param extraEnvVarsSecrets List of secrets with extra environment variables for all the Airflow pods\n##\nextraEnvVarsSecrets: []\n## @param sidecars Add additional sidecar containers to all the Airflow pods\n## Example:\n## sidecars:\n## - name: your-image-name\n## image: your-image\n## imagePullPolicy: Always\n## ports:\n## - name: portname\n## containerPort: 1234\n##\nsidecars: []\n## @param initContainers Add additional init containers to all the Airflow pods\n## Example:\n## initContainers:\n## - name: your-image-name\n## image: your-image\n## imagePullPolicy: Always\n## ports:\n## - name: portname\n## containerPort: 1234\n##\ninitContainers: []\n## @param extraVolumeMounts Optionally specify extra list of additional volumeMounts for all the Airflow pods\n##\nextraVolumeMounts: []\n## @param extraVolumes Optionally specify extra list of additional volumes for the all the Airflow pods\n##\nextraVolumes: []\n\n## @section Airflow web parameters\n\nweb:\n ## Bitnami Airflow image version\n ## ref: https://hub.docker.com/r/bitnami/airflow/tags/\n ## @param web.image.registry Airflow image registry\n ## @param web.image.repository Airflow image repository\n ## @param web.image.tag Airflow image tag (immutable tags are recommended)\n ## @param web.image.digest Airflow image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param web.image.pullPolicy Airflow image pull policy\n ## @param web.image.pullSecrets Airflow image pull secrets\n ## @param web.image.debug Enable image debug mode\n image:\n registry: docker.io\n repository: bitnami/airflow\n tag: 2.4.2-debian-11-r6\n digest: \"\"\n ## Specify a imagePullPolicy\n ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'\n ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images\n ##\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## Set to true if you would like to see extra information on logs\n ##\n debug: false\n ## @param web.baseUrl URL used to access to Airflow web ui\n ##\n baseUrl: \"\"\n ## @param web.existingConfigmap Name of an existing config map containing the Airflow web config file\n ##\n existingConfigmap: \"\"\n ## @param web.command Override default container command (useful when using custom images)\n ##\n command: []\n ## @param web.args Override default container args (useful when using custom images)\n ##\n args: []\n ## @param web.extraEnvVars Array with extra environment variables to add Airflow web pods\n ##\n extraEnvVars: []\n ## @param web.extraEnvVarsCM ConfigMap containing extra environment variables for Airflow web pods\n ##\n extraEnvVarsCM: \"\"\n ## @param web.extraEnvVarsSecret Secret containing extra environment variables (in case of sensitive data) for Airflow web pods\n ##\n extraEnvVarsSecret: \"\"\n ## @param web.extraEnvVarsSecrets List of secrets with extra environment variables for Airflow web pods\n ##\n extraEnvVarsSecrets: []\n ## @param web.containerPorts.http Airflow web HTTP container port\n ##\n containerPorts:\n http: 8080\n ## @param web.replicaCount Number of Airflow web replicas\n ##\n replicaCount: 1\n ## Configure extra options for Airflow web containers' liveness, readiness and startup probes\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes\n ## @param web.livenessProbe.enabled Enable livenessProbe on Airflow web containers\n ## @param web.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe\n ## @param web.livenessProbe.periodSeconds Period seconds for livenessProbe\n ## @param web.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe\n ## @param web.livenessProbe.failureThreshold Failure threshold for livenessProbe\n ## @param web.livenessProbe.successThreshold Success threshold for livenessProbe\n ##\n livenessProbe:\n enabled: true\n initialDelaySeconds: 180\n periodSeconds: 20\n timeoutSeconds: 5\n failureThreshold: 6\n successThreshold: 1\n ## @param web.readinessProbe.enabled Enable readinessProbe on Airflow web containers\n ## @param web.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe\n ## @param web.readinessProbe.periodSeconds Period seconds for readinessProbe\n ## @param web.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe\n ## @param web.readinessProbe.failureThreshold Failure threshold for readinessProbe\n ## @param web.readinessProbe.successThreshold Success threshold for readinessProbe\n ##\n readinessProbe:\n enabled: true\n initialDelaySeconds: 30\n periodSeconds: 10\n timeoutSeconds: 5\n failureThreshold: 6\n successThreshold: 1\n ## @param web.startupProbe.enabled Enable startupProbe on Airflow web containers\n ## @param web.startupProbe.initialDelaySeconds Initial delay seconds for startupProbe\n ## @param web.startupProbe.periodSeconds Period seconds for startupProbe\n ## @param web.startupProbe.timeoutSeconds Timeout seconds for startupProbe\n ## @param web.startupProbe.failureThreshold Failure threshold for startupProbe\n ## @param web.startupProbe.successThreshold Success threshold for startupProbe\n ##\n startupProbe:\n enabled: false\n initialDelaySeconds: 60\n periodSeconds: 10\n timeoutSeconds: 1\n failureThreshold: 15\n successThreshold: 1\n ## @param web.customLivenessProbe Custom livenessProbe that overrides the default one\n ##\n customLivenessProbe: {}\n ## @param web.customReadinessProbe Custom readinessProbe that overrides the default one\n ##\n customReadinessProbe: {}\n ## @param web.customStartupProbe Custom startupProbe that overrides the default one\n ##\n customStartupProbe: {}\n ## Airflow web resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ## @param web.resources.limits The resources limits for the Airflow web containers\n ## @param web.resources.requests The requested resources for the Airflow web containers\n ##\n resources:\n limits: {}\n requests: {}\n ## Configure Airflow web pods Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod\n ## @param web.podSecurityContext.enabled Enabled Airflow web pods' Security Context\n ## @param web.podSecurityContext.fsGroup Set Airflow web pod's Security Context fsGroup\n ##\n podSecurityContext:\n enabled: true\n fsGroup: 1001\n ## Configure Airflow web containers (only main one) Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container\n ## @param web.containerSecurityContext.enabled Enabled Airflow web containers' Security Context\n ## @param web.containerSecurityContext.runAsUser Set Airflow web containers' Security Context runAsUser\n ## @param web.containerSecurityContext.runAsNonRoot Set Airflow web containers' Security Context runAsNonRoot\n ##\n containerSecurityContext:\n enabled: true\n runAsUser: 1001\n runAsNonRoot: true\n ## @param web.lifecycleHooks for the Airflow web container(s) to automate configuration before or after startup\n ##\n lifecycleHooks: {}\n ## @param web.hostAliases Deployment pod host aliases\n ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/\n ##\n hostAliases: []\n ## @param web.podLabels Add extra labels to the Airflow web pods\n ##\n podLabels: {}\n ## @param web.podAnnotations Add extra annotations to the Airflow web pods\n ##\n podAnnotations: {}\n ## @param web.affinity Affinity for Airflow web pods assignment (evaluated as a template)\n ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity\n ## Note: `web.podAffinityPreset`, `web.podAntiAffinityPreset`, and `web.nodeAffinityPreset` will be ignored when it's set\n ##\n affinity: {}\n ## Node affinity preset\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity\n ## @param web.nodeAffinityPreset.key Node label key to match. Ignored if `web.affinity` is set.\n ## @param web.nodeAffinityPreset.type Node affinity preset type. Ignored if `web.affinity` is set. Allowed values: `soft` or `hard`\n ## @param web.nodeAffinityPreset.values Node label values to match. Ignored if `web.affinity` is set.\n ##\n nodeAffinityPreset:\n ## e.g:\n ## key: \"kubernetes.io/e2e-az-name\"\n ##\n key: \"\"\n type: \"\"\n ## e.g:\n ## values:\n ## - e2e-az1\n ## - e2e-az2\n ##\n values: []\n ## @param web.nodeSelector Node labels for Airflow web pods assignment\n ## Ref: https://kubernetes.io/docs/user-guide/node-selection/\n ##\n nodeSelector: {}\n ## @param web.podAffinityPreset Pod affinity preset. Ignored if `web.affinity` is set. Allowed values: `soft` or `hard`.\n ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAffinityPreset: \"\"\n ## @param web.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `web.affinity` is set. Allowed values: `soft` or `hard`.\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAntiAffinityPreset: soft\n ## @param web.tolerations Tolerations for Airflow web pods assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n ##\n tolerations: []\n ## @param web.topologySpreadConstraints Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template\n ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods\n ##\n topologySpreadConstraints: []\n ## @param web.priorityClassName Priority Class Name\n ## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass\n ##\n priorityClassName: \"\"\n ## @param web.schedulerName Use an alternate scheduler, e.g. \"stork\".\n ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/\n ##\n schedulerName: \"\"\n ## @param web.terminationGracePeriodSeconds Seconds Airflow web pod needs to terminate gracefully\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods\n ##\n terminationGracePeriodSeconds: \"\"\n ## @param web.updateStrategy.type Airflow web deployment strategy type\n ## @param web.updateStrategy.rollingUpdate Airflow web deployment rolling update configuration parameters\n ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy\n ##\n updateStrategy:\n type: RollingUpdate\n rollingUpdate: {}\n ## @param web.sidecars Add additional sidecar containers to the Airflow web pods\n ## Example:\n ## sidecars:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n sidecars: []\n ## @param web.initContainers Add additional init containers to the Airflow web pods\n ## Example:\n ## initContainers:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n initContainers: []\n ## @param web.extraVolumeMounts Optionally specify extra list of additional volumeMounts for the Airflow web pods\n ##\n extraVolumeMounts: []\n ## @param web.extraVolumes Optionally specify extra list of additional volumes for the Airflow web pods\n ##\n extraVolumes: []\n ## Airflow web Pod Disruption Budget\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/\n ## @param web.pdb.create Deploy a pdb object for the Airflow web pods\n ## @param web.pdb.minAvailable Maximum number/percentage of unavailable Airflow web replicas\n ## @param web.pdb.maxUnavailable Maximum number/percentage of unavailable Airflow web replicas\n ##\n pdb:\n create: false\n minAvailable: 1\n maxUnavailable: \"\"\n\n## @section Airflow scheduler parameters\n\nscheduler:\n ## Bitnami Airflow Scheduler image version\n ## ref: https://hub.docker.com/r/bitnami/airflow-scheduler/tags/\n ## @param scheduler.image.registry Airflow Scheduler image registry\n ## @param scheduler.image.repository Airflow Scheduler image repository\n ## @param scheduler.image.tag Airflow Scheduler image tag (immutable tags are recommended)\n ## @param scheduler.image.digest Airflow Schefuler image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param scheduler.image.pullPolicy Airflow Scheduler image pull policy\n ## @param scheduler.image.pullSecrets Airflow Scheduler image pull secrets\n ## @param scheduler.image.debug Enable image debug mode\n ##\n image:\n registry: docker.io\n repository: bitnami/airflow-scheduler\n tag: 2.4.2-debian-11-r4\n digest: \"\"\n ## Specify a imagePullPolicy\n ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'\n ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images\n ##\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## Set to true if you would like to see extra information on logs\n ##\n debug: false\n ## @param scheduler.replicaCount Number of scheduler replicas\n ##\n replicaCount: 1\n ## @param scheduler.command Override cmd\n ##\n command: []\n ## @param scheduler.args Override args\n ##\n args: []\n ## @param scheduler.extraEnvVars Add extra environment variables\n ##\n extraEnvVars: []\n ## @param scheduler.extraEnvVarsCM ConfigMap with extra environment variables\n ##\n extraEnvVarsCM: \"\"\n ## @param scheduler.extraEnvVarsSecret Secret with extra environment variables\n ##\n extraEnvVarsSecret: \"\"\n ## @param scheduler.extraEnvVarsSecrets List of secrets with extra environment variables for Airflow scheduler pods\n ##\n extraEnvVarsSecrets: []\n ## @param scheduler.customLivenessProbe Custom livenessProbe that overrides the default one\n ##\n customLivenessProbe: {}\n ## @param scheduler.customReadinessProbe Custom readinessProbe that overrides the default one\n ##\n customReadinessProbe: {}\n ## @param scheduler.customStartupProbe Custom startupProbe that overrides the default one\n ##\n customStartupProbe: {}\n ## Airflow scheduler resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ## @param scheduler.resources.limits The resources limits for the Airflow scheduler containers\n ## @param scheduler.resources.requests The requested resources for the Airflow scheduler containers\n ##\n resources:\n limits: {}\n requests: {}\n ## Configure Airflow scheduler pods Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod\n ## @param scheduler.podSecurityContext.enabled Enabled Airflow scheduler pods' Security Context\n ## @param scheduler.podSecurityContext.fsGroup Set Airflow scheduler pod's Security Context fsGroup\n ##\n podSecurityContext:\n enabled: true\n fsGroup: 1001\n ## Configure Airflow scheduler containers (only main one) Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container\n ## @param scheduler.containerSecurityContext.enabled Enabled Airflow scheduler containers' Security Context\n ## @param scheduler.containerSecurityContext.runAsUser Set Airflow scheduler containers' Security Context runAsUser\n ## @param scheduler.containerSecurityContext.runAsNonRoot Set Airflow scheduler containers' Security Context runAsNonRoot\n ##\n containerSecurityContext:\n enabled: true\n runAsUser: 1001\n runAsNonRoot: true\n ## @param scheduler.lifecycleHooks for the Airflow scheduler container(s) to automate configuration before or after startup\n ##\n lifecycleHooks: {}\n ## @param scheduler.hostAliases Deployment pod host aliases\n ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/\n ##\n hostAliases: []\n ## @param scheduler.podLabels Add extra labels to the Airflow scheduler pods\n ##\n podLabels: {}\n ## @param scheduler.podAnnotations Add extra annotations to the Airflow scheduler pods\n ##\n podAnnotations: {}\n ## @param scheduler.affinity Affinity for Airflow scheduler pods assignment (evaluated as a template)\n ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity\n ## Note: `scheduler.podAffinityPreset`, `scheduler.podAntiAffinityPreset`, and `scheduler.nodeAffinityPreset` will be ignored when it's set\n ##\n affinity: {}\n ## Node affinity preset\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity\n ## @param scheduler.nodeAffinityPreset.key Node label key to match. Ignored if `scheduler.affinity` is set.\n ## @param scheduler.nodeAffinityPreset.type Node affinity preset type. Ignored if `scheduler.affinity` is set. Allowed values: `soft` or `hard`\n ## @param scheduler.nodeAffinityPreset.values Node label values to match. Ignored if `scheduler.affinity` is set.\n ##\n nodeAffinityPreset:\n ## e.g:\n ## key: \"kubernetes.io/e2e-az-name\"\n ##\n key: \"\"\n type: \"\"\n ## e.g:\n ## values:\n ## - e2e-az1\n ## - e2e-az2\n ##\n values: []\n ## @param scheduler.nodeSelector Node labels for Airflow scheduler pods assignment\n ## Ref: https://kubernetes.io/docs/user-guide/node-selection/\n ##\n nodeSelector: {}\n ## @param scheduler.podAffinityPreset Pod affinity preset. Ignored if `scheduler.affinity` is set. Allowed values: `soft` or `hard`.\n ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAffinityPreset: \"\"\n ## @param scheduler.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `scheduler.affinity` is set. Allowed values: `soft` or `hard`.\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAntiAffinityPreset: soft\n ## @param scheduler.tolerations Tolerations for Airflow scheduler pods assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n ##\n tolerations: []\n ## @param scheduler.topologySpreadConstraints Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template\n ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods\n ##\n topologySpreadConstraints: []\n ## @param scheduler.priorityClassName Priority Class Name\n ## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass\n ##\n priorityClassName: \"\"\n ## @param scheduler.schedulerName Use an alternate scheduler, e.g. \"stork\".\n ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/\n ##\n schedulerName: \"\"\n ## @param scheduler.terminationGracePeriodSeconds Seconds Airflow scheduler pod needs to terminate gracefully\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods\n ##\n terminationGracePeriodSeconds: \"\"\n ## @param scheduler.updateStrategy.type Airflow scheduler deployment strategy type\n ## @param scheduler.updateStrategy.rollingUpdate Airflow scheduler deployment rolling update configuration parameters\n ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy\n ##\n updateStrategy:\n type: RollingUpdate\n rollingUpdate: {}\n ## @param scheduler.sidecars Add additional sidecar containers to the Airflow scheduler pods\n ## Example:\n ## sidecars:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n sidecars: []\n ## @param scheduler.initContainers Add additional init containers to the Airflow scheduler pods\n ## Example:\n ## initContainers:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n initContainers: []\n ## @param scheduler.extraVolumeMounts Optionally specify extra list of additional volumeMounts for the Airflow scheduler pods\n ##\n extraVolumeMounts: []\n ## @param scheduler.extraVolumes Optionally specify extra list of additional volumes for the Airflow scheduler pods\n ##\n extraVolumes: []\n ## Airflow scheduler Pod Disruption Budget\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/\n ## @param scheduler.pdb.create Deploy a pdb object for the Airflow scheduler pods\n ## @param scheduler.pdb.minAvailable Maximum number/percentage of unavailable Airflow scheduler replicas\n ## @param scheduler.pdb.maxUnavailable Maximum number/percentage of unavailable Airflow scheduler replicas\n ##\n pdb:\n create: false\n minAvailable: 1\n maxUnavailable: \"\"\n\n## @section Airflow worker parameters\n\nworker:\n ## Bitnami Airflow Worker image version\n ## ref: https://hub.docker.com/r/bitnami/airflow-worker/tags/\n ## @param worker.image.registry Airflow Worker image registry\n ## @param worker.image.repository Airflow Worker image repository\n ## @param worker.image.tag Airflow Worker image tag (immutable tags are recommended)\n ## @param worker.image.digest Airflow Worker image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param worker.image.pullPolicy Airflow Worker image pull policy\n ## @param worker.image.pullSecrets Airflow Worker image pull secrets\n ## @param worker.image.debug Enable image debug mode\n ##\n image:\n registry: docker.io\n repository: bitnami/airflow-worker\n tag: 2.4.2-debian-11-r4\n digest: \"\"\n ## Specify a imagePullPolicy\n ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'\n ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images\n ##\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## Set to true if you would like to see extra information on logs\n ##\n debug: false\n ## @param worker.command Override default container command (useful when using custom images)\n ##\n command: []\n ## @param worker.args Override default container args (useful when using custom images)\n ##\n args: []\n ## @param worker.extraEnvVars Array with extra environment variables to add Airflow worker pods\n ##\n extraEnvVars: []\n ## @param worker.extraEnvVarsCM ConfigMap containing extra environment variables for Airflow worker pods\n ##\n extraEnvVarsCM: \"\"\n ## @param worker.extraEnvVarsSecret Secret containing extra environment variables (in case of sensitive data) for Airflow worker pods\n ##\n extraEnvVarsSecret: \"\"\n ## @param worker.extraEnvVarsSecrets List of secrets with extra environment variables for Airflow worker pods\n ##\n extraEnvVarsSecrets: []\n ## @param worker.containerPorts.http Airflow worker HTTP container port\n ##\n containerPorts:\n http: 8793\n ## @param worker.replicaCount Number of Airflow worker replicas\n ##\n replicaCount: 1\n ## Configure extra options for Airflow worker containers' liveness, readiness and startup probes\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes\n ## @param worker.livenessProbe.enabled Enable livenessProbe on Airflow worker containers\n ## @param worker.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe\n ## @param worker.livenessProbe.periodSeconds Period seconds for livenessProbe\n ## @param worker.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe\n ## @param worker.livenessProbe.failureThreshold Failure threshold for livenessProbe\n ## @param worker.livenessProbe.successThreshold Success threshold for livenessProbe\n ##\n livenessProbe:\n enabled: true\n initialDelaySeconds: 180\n periodSeconds: 20\n timeoutSeconds: 5\n failureThreshold: 6\n successThreshold: 1\n ## @param worker.readinessProbe.enabled Enable readinessProbe on Airflow worker containers\n ## @param worker.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe\n ## @param worker.readinessProbe.periodSeconds Period seconds for readinessProbe\n ## @param worker.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe\n ## @param worker.readinessProbe.failureThreshold Failure threshold for readinessProbe\n ## @param worker.readinessProbe.successThreshold Success threshold for readinessProbe\n ##\n readinessProbe:\n enabled: true\n initialDelaySeconds: 30\n periodSeconds: 10\n timeoutSeconds: 5\n failureThreshold: 6\n successThreshold: 1\n ## @param worker.startupProbe.enabled Enable startupProbe on Airflow worker containers\n ## @param worker.startupProbe.initialDelaySeconds Initial delay seconds for startupProbe\n ## @param worker.startupProbe.periodSeconds Period seconds for startupProbe\n ## @param worker.startupProbe.timeoutSeconds Timeout seconds for startupProbe\n ## @param worker.startupProbe.failureThreshold Failure threshold for startupProbe\n ## @param worker.startupProbe.successThreshold Success threshold for startupProbe\n ##\n startupProbe:\n enabled: false\n initialDelaySeconds: 60\n periodSeconds: 10\n timeoutSeconds: 1\n failureThreshold: 15\n successThreshold: 1\n ## @param worker.customLivenessProbe Custom livenessProbe that overrides the default one\n ##\n customLivenessProbe: {}\n ## @param worker.customReadinessProbe Custom readinessProbe that overrides the default one\n ##\n customReadinessProbe: {}\n ## @param worker.customStartupProbe Custom startupProbe that overrides the default one\n ##\n customStartupProbe: {}\n ## Airflow worker resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ## @param worker.resources.limits The resources limits for the Airflow worker containers\n ## @param worker.resources.requests The requested resources for the Airflow worker containers\n ##\n resources:\n limits: {}\n requests: {}\n ## Configure Airflow worker pods Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod\n ## @param worker.podSecurityContext.enabled Enabled Airflow worker pods' Security Context\n ## @param worker.podSecurityContext.fsGroup Set Airflow worker pod's Security Context fsGroup\n ##\n podSecurityContext:\n enabled: true\n fsGroup: 1001\n ## Configure Airflow worker containers (only main one) Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container\n ## @param worker.containerSecurityContext.enabled Enabled Airflow worker containers' Security Context\n ## @param worker.containerSecurityContext.runAsUser Set Airflow worker containers' Security Context runAsUser\n ## @param worker.containerSecurityContext.runAsNonRoot Set Airflow worker containers' Security Context runAsNonRoot\n ##\n containerSecurityContext:\n enabled: true\n runAsUser: 1001\n runAsNonRoot: true\n ## @param worker.lifecycleHooks for the Airflow worker container(s) to automate configuration before or after startup\n ##\n lifecycleHooks: {}\n ## @param worker.hostAliases Deployment pod host aliases\n ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/\n ##\n hostAliases: []\n ## @param worker.podLabels Add extra labels to the Airflow worker pods\n ##\n podLabels: {}\n ## @param worker.podAnnotations Add extra annotations to the Airflow worker pods\n ##\n podAnnotations: {}\n ## @param worker.affinity Affinity for Airflow worker pods assignment (evaluated as a template)\n ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity\n ## Note: `worker.podAffinityPreset`, `worker.podAntiAffinityPreset`, and `worker.nodeAffinityPreset` will be ignored when it's set\n ##\n affinity: {}\n ## Node affinity preset\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity\n ## @param worker.nodeAffinityPreset.key Node label key to match. Ignored if `worker.affinity` is set.\n ## @param worker.nodeAffinityPreset.type Node affinity preset type. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard`\n ## @param worker.nodeAffinityPreset.values Node label values to match. Ignored if `worker.affinity` is set.\n ##\n nodeAffinityPreset:\n ## e.g:\n ## key: \"kubernetes.io/e2e-az-name\"\n ##\n key: \"\"\n type: \"\"\n ## e.g:\n ## values:\n ## - e2e-az1\n ## - e2e-az2\n ##\n values: []\n ## @param worker.nodeSelector Node labels for Airflow worker pods assignment\n ## Ref: https://kubernetes.io/docs/user-guide/node-selection/\n ##\n nodeSelector: {}\n ## @param worker.podAffinityPreset Pod affinity preset. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard`.\n ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAffinityPreset: \"\"\n ## @param worker.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard`.\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAntiAffinityPreset: soft\n ## @param worker.tolerations Tolerations for Airflow worker pods assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n ##\n tolerations: []\n ## @param worker.topologySpreadConstraints Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template\n ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods\n ##\n topologySpreadConstraints: []\n ## @param worker.priorityClassName Priority Class Name\n ## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass\n ##\n priorityClassName: \"\"\n ## @param worker.schedulerName Use an alternate scheduler, e.g. \"stork\".\n ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/\n ##\n schedulerName: \"\"\n ## @param worker.terminationGracePeriodSeconds Seconds Airflow worker pod needs to terminate gracefully\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods\n ##\n terminationGracePeriodSeconds: \"\"\n ## @param worker.updateStrategy.type Airflow worker deployment strategy type\n ## @param worker.updateStrategy.rollingUpdate Airflow worker deployment rolling update configuration parameters\n ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy\n ##\n updateStrategy:\n type: RollingUpdate\n rollingUpdate: {}\n ## @param worker.sidecars Add additional sidecar containers to the Airflow worker pods\n ## Example:\n ## sidecars:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n sidecars: []\n ## @param worker.initContainers Add additional init containers to the Airflow worker pods\n ## Example:\n ## initContainers:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n initContainers: []\n ## @param worker.extraVolumeMounts Optionally specify extra list of additional volumeMounts for the Airflow worker pods\n ##\n extraVolumeMounts: []\n ## @param worker.extraVolumes Optionally specify extra list of additional volumes for the Airflow worker pods\n ##\n extraVolumes: []\n ## @param worker.extraVolumeClaimTemplates Optionally specify extra list of volumesClaimTemplates for the Airflow worker statefulset\n ##\n extraVolumeClaimTemplates: []\n ## @param worker.podTemplate Template to replace the default one to be use when `executor=KubernetesExecutor` to create Airflow worker pods\n ##\n podTemplate: {}\n ## Airflow worker Pod Disruption Budget\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/\n ## @param worker.pdb.create Deploy a pdb object for the Airflow worker pods\n ## @param worker.pdb.minAvailable Maximum number/percentage of unavailable Airflow worker replicas\n ## @param worker.pdb.maxUnavailable Maximum number/percentage of unavailable Airflow worker replicas\n ##\n pdb:\n create: false\n minAvailable: 1\n maxUnavailable: \"\"\n ## Enable HorizontalPodAutoscaler for worker pods\n ## ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/\n ## @param worker.autoscaling.enabled Whether enable horizontal pod autoscaler\n ## @param worker.autoscaling.minReplicas Configure a minimum amount of pods\n ## @param worker.autoscaling.maxReplicas Configure a maximum amount of pods\n ## @param worker.autoscaling.targetCPU Define the CPU target to trigger the scaling actions (utilization percentage)\n ## @param worker.autoscaling.targetMemory Define the memory target to trigger the scaling actions (utilization percentage)\n ##\n autoscaling:\n enabled: false\n minReplicas: 1\n maxReplicas: 3\n targetCPU: 80\n targetMemory: 80\n\n## @section Airflow git sync parameters\n\n## Configure Git to pull dags and plugins\n##\ngit:\n ## Bitnami Git image version\n ## ref: https://hub.docker.com/r/bitnami/git/tags/\n ## @param git.image.registry Git image registry\n ## @param git.image.repository Git image repository\n ## @param git.image.tag Git image tag (immutable tags are recommended)\n ## @param git.image.digest Git image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param git.image.pullPolicy Git image pull policy\n ## @param git.image.pullSecrets Git image pull secrets\n ##\n image:\n registry: docker.io\n repository: bitnami/git\n tag: 2.38.1-debian-11-r7\n digest: \"\"\n ## Specify a imagePullPolicy\n ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'\n ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images\n ##\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## Get DAG files from git repositories\n ## @param git.dags.enabled Enable in order to download DAG files from git repositories.\n ## @param git.dags.repositories [array] Array of repositories from which to download DAG files\n ##\n dags:\n enabled: false\n ## Name for repositories can be anything unique and must follow same naming conventions as kubernetes.\n ## Kubernetes resources can have names up to 253 characters long. The characters allowed in names are:\n ## digits (0-9), lower case letters (a-z), -, and .\n ## Example:\n ## - repository: https://github.com/myuser/myrepo\n ## branch: main\n ## name: my-dags\n ## path: /\n ##\n repositories:\n - repository: \"\"\n ## Branch from repository to checkout\n ##\n branch: \"\"\n ## An unique identifier for repository, must be unique for each repository\n ##\n name: \"\"\n ## Path to a folder in the repository containing the dags\n ##\n path: \"\"\n ## Get Plugins files from git repositories.\n ## @param git.plugins.enabled Enable in order to download Plugins files from git repositories.\n ## @param git.plugins.repositories [array] Array of repositories from which to download DAG files\n ##\n plugins:\n enabled: false\n repositories:\n - repository: \"\"\n ## Branch from repository to checkout\n ##\n branch: \"\"\n ## An unique identifier for repository, must be unique for each repository\n ##\n name: \"\"\n ## Path to a folder in the repository containing the plugins\n ##\n path: \"\"\n ## Properties for the Clone init container\n ## @param git.clone.command Override cmd\n ## @param git.clone.args Override args\n ## @param git.clone.extraVolumeMounts Add extra volume mounts\n ## @param git.clone.extraEnvVars Add extra environment variables\n ## @param git.clone.extraEnvVarsCM ConfigMap with extra environment variables\n ## @param git.clone.extraEnvVarsSecret Secret with extra environment variables\n ## @param git.clone.resources Clone init container resource requests and limits\n ##\n clone:\n command: []\n args: []\n extraVolumeMounts: []\n extraEnvVars: []\n extraEnvVarsCM: \"\"\n extraEnvVarsSecret: \"\"\n ## Clone init container resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ##\n resources: {}\n ## Properties for the Sync sidecar container\n ## @param git.sync.interval Interval in seconds to pull the git repository containing the plugins and/or DAG files\n ## @param git.sync.command Override cmd\n ## @param git.sync.args Override args\n ## @param git.sync.extraVolumeMounts Add extra volume mounts\n ## @param git.sync.extraEnvVars Add extra environment variables\n ## @param git.sync.extraEnvVarsCM ConfigMap with extra environment variables\n ## @param git.sync.extraEnvVarsSecret Secret with extra environment variables\n ## @param git.sync.resources Sync sidecar container resource requests and limits\n ##\n sync:\n interval: 60\n command: []\n args: []\n extraVolumeMounts: []\n extraEnvVars: []\n extraEnvVarsCM: \"\"\n extraEnvVarsSecret: \"\"\n ## Sync sidecar container resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ##\n resources: {}\n\n## @section Airflow ldap parameters\n\n## LDAP configuration\n## @param ldap.enabled Enable LDAP authentication\n## @param ldap.uri Server URI, eg. ldap://ldap_server:389\n## DEPRECATED ldap.base It will be removed in a future release, please use 'ldap.basedn' instead\n## @param ldap.basedn Base of the search, eg. ou=example,o=org.\n## DEPRECATED ldap.uidField It will be removed in a future release,, please use 'ldap.searchAttribute' instead\n## @param ldap.searchAttribute if doing an indirect bind to ldap, this is the field that matches the username when searching for the account to bind to\n## @param ldap.binddn DN of the account used to search in the LDAP server.\n## @param ldap.bindpw Bind Password\n## @param ldap.userRegistration Set to True to enable user self registration\n## @param ldap.userRegistrationRole Set role name to be assign when a user registers himself. This role must already exist. Mandatory when using ldap.userRegistration\n## @param ldap.rolesMapping mapping from LDAP DN to a list of roles\n## @param ldap.rolesSyncAtLogin replace ALL the user's roles each login, or only on registration\n##\nldap:\n enabled: false\n uri: \"ldap://ldap_server:389\"\n basedn: \"dc=example,dc=org\"\n searchAttribute: \"cn\"\n binddn: \"cn=admin,dc=example,dc=org\"\n bindpw: \"\"\n userRegistration: 'True'\n userRegistrationRole: \"Public\"\n rolesMapping: '{ \"cn=All,ou=Groups,dc=example,dc=org\": [\"User\"], \"cn=Admins,ou=Groups,dc=example,dc=org\": [\"Admin\"], }'\n rolesSyncAtLogin: 'True'\n\n ## SSL/TLS parameters for LDAP\n ## @param ldap.tls.enabled Enabled TLS/SSL for LDAP, you must include the CA file.\n ## @param ldap.tls.allowSelfSigned Allow to use self signed certificates\n ## DEPRECATED ldap.tls.CAcertificateSecret It will be removed in a future release, please use ldap.tls.certificatesSecret instead\n ## @param ldap.tls.certificatesSecret Name of the existing secret containing the certificate CA file that will be used by ldap client\n ## @param ldap.tls.certificatesMountPath Where LDAP certifcates are mounted.\n ## DEPRECATED ldap.tls.CAcertificateFilename It will be removed in a future release, please use ldap.tls.CAFilename instead\n ## @param ldap.tls.CAFilename LDAP CA cert filename\n ##\n tls:\n enabled: false\n allowSelfSigned: true\n certificatesSecret: \"\"\n certificatesMountPath: /opt/bitnami/airflow/conf/certs\n CAFilename: \"\"\n\n## @section Traffic Exposure Parameters\n\n## Airflow service parameters\n##\nservice:\n ## @param service.type Airflow service type\n ##\n type: ClusterIP\n ## @param service.ports.http Airflow service HTTP port\n ##\n ports:\n http: 8080\n ## Node ports to expose\n ## @param service.nodePorts.http Node port for HTTP\n ## NOTE: choose port between <30000-32767>\n ##\n nodePorts:\n http: \"\"\n ## @param service.sessionAffinity Control where client requests go, to the same pod or round-robin\n ## Values: ClientIP or None\n ## ref: https://kubernetes.io/docs/user-guide/services/\n ##\n sessionAffinity: None\n ## @param service.sessionAffinityConfig Additional settings for the sessionAffinity\n ## sessionAffinityConfig:\n ## clientIP:\n ## timeoutSeconds: 300\n ##\n sessionAffinityConfig: {}\n ## @param service.clusterIP Airflow service Cluster IP\n ## e.g.:\n ## clusterIP: None\n ##\n clusterIP: \"\"\n ## @param service.loadBalancerIP Airflow service Load Balancer IP\n ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-loadbalancer\n ##\n loadBalancerIP: \"\"\n ## @param service.loadBalancerSourceRanges Airflow service Load Balancer sources\n ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service\n ## e.g:\n ## loadBalancerSourceRanges:\n ## - 10.10.10.0/24\n ##\n loadBalancerSourceRanges: []\n ## @param service.externalTrafficPolicy Airflow service external traffic policy\n ## ref https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip\n ##\n externalTrafficPolicy: Cluster\n ## @param service.annotations Additional custom annotations for Airflow service\n ##\n annotations: {}\n ## @param service.extraPorts Extra port to expose on Airflow service\n ##\n extraPorts: []\n\n## Airflow ingress parameters\n## ref: https://kubernetes.io/docs/user-guide/ingress/\n##\ningress:\n ## @param ingress.enabled Enable ingress record generation for Airflow\n ##\n enabled: false\n ## @param ingress.ingressClassName IngressClass that will be be used to implement the Ingress (Kubernetes 1.18+)\n ## This is supported in Kubernetes 1.18+ and required if you have more than one IngressClass marked as the default for your cluster .\n ## ref: https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/\n ##\n ingressClassName: \"\"\n ## @param ingress.pathType Ingress path type\n ##\n pathType: ImplementationSpecific\n ## @param ingress.apiVersion Force Ingress API version (automatically detected if not set)\n ##\n apiVersion: \"\"\n ## @param ingress.hostname Default host for the ingress record\n ##\n hostname: airflow.local\n ## @param ingress.path Default path for the ingress record\n ## NOTE: You may need to set this to '/*' in order to use this with ALB ingress controllers\n ##\n path: /\n ## @param ingress.annotations [object] Additional annotations for the Ingress resource. To enable certificate autogeneration, place here your cert-manager annotations.\n ## Use this parameter to set the required annotations for cert-manager, see\n ## ref: https://cert-manager.io/docs/usage/ingress/#supported-annotations\n ## e.g:\n ## annotations:\n ## kubernetes.io/ingress.class: nginx\n ## cert-manager.io/cluster-issuer: cluster-issuer-name\n ##\n annotations: {}\n ## @param ingress.tls Enable TLS configuration for the host defined at `ingress.hostname` parameter\n ## TLS certificates will be retrieved from a TLS secret with name: `{{- printf \"%s-tls\" .Values.ingress.hostname }}`\n ## You can:\n ## - Use the `ingress.secrets` parameter to create this TLS secret\n ## - Rely on cert-manager to create it by setting the corresponding annotations\n ## - Rely on Helm to create self-signed certificates by setting `ingress.selfSigned=true`\n ##\n tls: false\n ## @param ingress.selfSigned Create a TLS secret for this ingress record using self-signed certificates generated by Helm\n ##\n selfSigned: false\n ## @param ingress.extraHosts An array with additional hostname(s) to be covered with the ingress record\n ## e.g:\n ## extraHosts:\n ## - name: airflow.local\n ## path: /\n ##\n extraHosts: []\n ## @param ingress.extraPaths An array with additional arbitrary paths that may need to be added to the ingress under the main host\n ## e.g:\n ## extraPaths:\n ## - path: /*\n ## backend:\n ## serviceName: ssl-redirect\n ## servicePort: use-annotation\n ##\n extraPaths: []\n ## @param ingress.extraTls TLS configuration for additional hostname(s) to be covered with this ingress record\n ## ref: https://kubernetes.io/docs/concepts/services-networking/ingress/#tls\n ## e.g:\n ## extraTls:\n ## - hosts:\n ## - airflow.local\n ## secretName: airflow.local-tls\n ##\n extraTls: []\n ## @param ingress.secrets Custom TLS certificates as secrets\n ## NOTE: 'key' and 'certificate' are expected in PEM format\n ## NOTE: 'name' should line up with a 'secretName' set further up\n ## If it is not set and you're using cert-manager, this is unneeded, as it will create a secret for you with valid certificates\n ## If it is not set and you're NOT using cert-manager either, self-signed certificates will be created valid for 365 days\n ## It is also possible to create and manage the certificates outside of this helm chart\n ## Please see README.md for more information\n ## e.g:\n ## secrets:\n ## - name: airflow.local-tls\n ## key: |-\n ## -----BEGIN RSA PRIVATE KEY-----\n ## ...\n ## -----END RSA PRIVATE KEY-----\n ## certificate: |-\n ## -----BEGIN CERTIFICATE-----\n ## ...\n ## -----END CERTIFICATE-----\n ##\n secrets: []\n ## @param ingress.extraRules Additional rules to be covered with this ingress record\n ## ref: https://kubernetes.io/docs/concepts/services-networking/ingress/#ingress-rules\n ## e.g:\n ## extraRules:\n ## - host: example.local\n ## http:\n ## path: /\n ## backend:\n ## service:\n ## name: example-svc\n ## port:\n ## name: http\n ##\n extraRules: []\n\n## @section Other Parameters\n\n## Service account for Airflow pods to use.\n## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/\n##\nserviceAccount:\n ## @param serviceAccount.create Enable creation of ServiceAccount for Airflow pods\n ##\n create: false\n ## @param serviceAccount.name The name of the ServiceAccount to use.\n ## If not set and create is true, a name is generated using the common.names.fullname template\n ##\n name: \"\"\n ## @param serviceAccount.automountServiceAccountToken Allows auto mount of ServiceAccountToken on the serviceAccount created\n ## Can be set to false if pods using this serviceAccount do not need to use K8s API\n ##\n automountServiceAccountToken: true\n ## @param serviceAccount.annotations Additional custom annotations for the ServiceAccount\n ##\n annotations: {}\n## Role Based Access\n## Ref: https://kubernetes.io/docs/admin/authorization/rbac/\n## @param rbac.create Create Role and RoleBinding\n##\nrbac:\n create: false\n ## @param rbac.rules Custom RBAC rules to set\n ## e.g:\n ## rules:\n ## - apiGroups:\n ## - \"\"\n ## resources:\n ## - pods\n ## verbs:\n ## - get\n ## - list\n ##\n rules: []\n\n## @section Airflow metrics parameters\n\nmetrics:\n ## @param metrics.enabled Whether or not to create a standalone Airflow exporter to expose Airflow metrics\n ##\n enabled: false\n ## Bitnami Airflow exporter image\n ## ref: https://hub.docker.com/r/bitnami/airflow-exporter/tags/\n ## @param metrics.image.registry Airflow exporter image registry\n ## @param metrics.image.repository Airflow exporter image repository\n ## @param metrics.image.tag Airflow exporter image tag (immutable tags are recommended)\n ## @param metrics.image.digest Airflow exporter image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param metrics.image.pullPolicy Airflow exporter image pull policy\n ## @param metrics.image.pullSecrets Airflow exporter image pull secrets\n ##\n image:\n registry: docker.io\n repository: bitnami/airflow-exporter\n tag: 0.20220314.0-debian-11-r57\n digest: \"\"\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## @param metrics.extraEnvVars Array with extra environment variables to add Airflow exporter pods\n ##\n extraEnvVars: []\n ## @param metrics.extraEnvVarsCM ConfigMap containing extra environment variables for Airflow exporter pods\n ##\n extraEnvVarsCM: \"\"\n ## @param metrics.extraEnvVarsSecret Secret containing extra environment variables (in case of sensitive data) for Airflow exporter pods\n ##\n extraEnvVarsSecret: \"\"\n ## @param metrics.containerPorts.http Airflow exporter metrics container port\n ##\n containerPorts:\n http: 9112\n ## Airflow exporter resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ## @param metrics.resources.limits The resources limits for the container\n ## @param metrics.resources.requests The requested resources for the container\n ##\n resources:\n limits: {}\n requests: {}\n ## Airflow exporter pods' Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod\n ## @param metrics.podSecurityContext.enabled Enable security context for the pods\n ## @param metrics.podSecurityContext.fsGroup Set Airflow exporter pod's Security Context fsGroup\n ##\n podSecurityContext:\n enabled: true\n fsGroup: 1001\n ## Airflow exporter containers' Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container\n ## @param metrics.containerSecurityContext.enabled Enable Airflow exporter containers' Security Context\n ## @param metrics.containerSecurityContext.runAsUser Set Airflow exporter containers' Security Context runAsUser\n ## @param metrics.containerSecurityContext.runAsNonRoot Set Airflow exporter containers' Security Context runAsNonRoot\n ## e.g:\n ## containerSecurityContext:\n ## enabled: true\n ## capabilities:\n ## drop: [\"NET_RAW\"]\n ## readOnlyRootFilesystem: true\n ##\n containerSecurityContext:\n enabled: true\n runAsUser: 1001\n runAsNonRoot: true\n ## @param metrics.lifecycleHooks for the Airflow exporter container(s) to automate configuration before or after startup\n ##\n lifecycleHooks: {}\n ## @param metrics.hostAliases Airflow exporter pods host aliases\n ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/\n ##\n hostAliases: []\n ## @param metrics.podLabels Extra labels for Airflow exporter pods\n ## Ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/\n ##\n podLabels: {}\n ## @param metrics.podAnnotations Extra annotations for Airflow exporter pods\n ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/\n ##\n podAnnotations: {}\n ## @param metrics.podAffinityPreset Pod affinity preset. Ignored if `metrics.affinity` is set. Allowed values: `soft` or `hard`\n ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAffinityPreset: \"\"\n ## @param metrics.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `metrics.affinity` is set. Allowed values: `soft` or `hard`\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAntiAffinityPreset: soft\n ## Node metrics.affinity preset\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity\n ##\n nodeAffinityPreset:\n ## @param metrics.nodeAffinityPreset.type Node affinity preset type. Ignored if `metrics.affinity` is set. Allowed values: `soft` or `hard`\n ##\n type: \"\"\n ## @param metrics.nodeAffinityPreset.key Node label key to match Ignored if `metrics.affinity` is set.\n ## E.g.\n ## key: \"kubernetes.io/e2e-az-name\"\n ##\n key: \"\"\n ## @param metrics.nodeAffinityPreset.values Node label values to match. Ignored if `metrics.affinity` is set.\n ## E.g.\n ## values:\n ## - e2e-az1\n ## - e2e-az2\n ##\n values: []\n ## @param metrics.affinity Affinity for pod assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity\n ## Note: metrics.podAffinityPreset, metrics.podAntiAffinityPreset, and metrics.nodeAffinityPreset will be ignored when it's set\n ##\n affinity: {}\n ## @param metrics.nodeSelector Node labels for pod assignment\n ## Ref: https://kubernetes.io/docs/user-guide/node-selection/\n ##\n nodeSelector: {}\n ## @param metrics.tolerations Tolerations for pod assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n ##\n tolerations: []\n ## @param metrics.schedulerName Name of the k8s scheduler (other than default) for Airflow exporter\n ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/\n ##\n schedulerName: \"\"\n ## Airflow exporter service configuration\n ##\n service:\n ## @param metrics.service.ports.http Airflow exporter metrics service port\n ##\n ports:\n http: 9112\n ## @param metrics.service.clusterIP Static clusterIP or None for headless services\n ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#choosing-your-own-ip-address\n ##\n clusterIP: \"\"\n ## @param metrics.service.sessionAffinity Control where client requests go, to the same pod or round-robin\n ## Values: ClientIP or None\n ## ref: https://kubernetes.io/docs/user-guide/services/\n ##\n sessionAffinity: None\n ## @param metrics.service.annotations [object] Annotations for the Airflow exporter service\n ##\n annotations:\n prometheus.io/scrape: \"true\"\n prometheus.io/port: \"{{ .Values.metrics.service.ports.http }}\"\n ## Prometheus Operator ServiceMonitor configuration\n ##\n serviceMonitor:\n ## @param metrics.serviceMonitor.enabled if `true`, creates a Prometheus Operator ServiceMonitor (requires `metrics.enabled` to be `true`)\n ##\n enabled: false\n ## @param metrics.serviceMonitor.namespace Namespace in which Prometheus is running\n ##\n namespace: \"\"\n ## @param metrics.serviceMonitor.interval Interval at which metrics should be scraped\n ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint\n ##\n interval: \"\"\n ## @param metrics.serviceMonitor.scrapeTimeout Timeout after which the scrape is ended\n ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint\n ##\n scrapeTimeout: \"\"\n ## @param metrics.serviceMonitor.labels Additional labels that can be used so ServiceMonitor will be discovered by Prometheus\n ##\n labels: {}\n ## @param metrics.serviceMonitor.selector Prometheus instance selector labels\n ## ref: https://github.com/bitnami/charts/tree/main/bitnami/prometheus-operator#prometheus-configuration\n ##\n selector: {}\n ## @param metrics.serviceMonitor.relabelings RelabelConfigs to apply to samples before scraping\n ##\n relabelings: []\n ## @param metrics.serviceMonitor.metricRelabelings MetricRelabelConfigs to apply to samples before ingestion\n ##\n metricRelabelings: []\n ## @param metrics.serviceMonitor.honorLabels Specify honorLabels parameter to add the scrape endpoint\n ##\n honorLabels: false\n ## @param metrics.serviceMonitor.jobLabel The name of the label on the target service to use as the job name in prometheus.\n ##\n jobLabel: \"\"\n\n## @section Airflow database parameters\n\n## PostgreSQL chart configuration\n## ref: https://github.com/bitnami/charts/blob/main/bitnami/postgresql/values.yaml\n## @param postgresql.enabled Switch to enable or disable the PostgreSQL helm chart\n## @param postgresql.auth.enablePostgresUser Assign a password to the \"postgres\" admin user. Otherwise, remote access will be blocked for this user\n## @param postgresql.auth.username Name for a custom user to create\n## @param postgresql.auth.password Password for the custom user to create\n## @param postgresql.auth.database Name for a custom database to create\n## @param postgresql.auth.existingSecret Name of existing secret to use for PostgreSQL credentials\n## @param postgresql.architecture PostgreSQL architecture (`standalone` or `replication`)\n##\npostgresql:\n enabled: true\n auth:\n enablePostgresUser: false\n username: bn_airflow\n password: \"\"\n database: bitnami_airflow\n existingSecret: \"\"\n architecture: standalone\n## External PostgreSQL configuration\n## All of these values are only used when postgresql.enabled is set to false\n## @param externalDatabase.host Database host\n## @param externalDatabase.port Database port number\n## @param externalDatabase.user Non-root username for Airflow\n## @param externalDatabase.password Password for the non-root username for Airflow\n## @param externalDatabase.database Airflow database name\n## @param externalDatabase.existingSecret Name of an existing secret resource containing the database credentials\n## @param externalDatabase.existingSecretPasswordKey Name of an existing secret key containing the database credentials\n##\nexternalDatabase:\n host: localhost\n port: 5432\n user: bn_airflow\n database: bitnami_airflow\n password: \"\"\n existingSecret: \"\"\n existingSecretPasswordKey: \"\"\n\n## Redis® chart configuration\n## ref: https://github.com/bitnami/charts/blob/main/bitnami/redis/values.yaml\n## @param redis.enabled Switch to enable or disable the Redis® helm\n## @param redis.auth.enabled Enable password authentication\n## @param redis.auth.password Redis® password\n## @param redis.auth.existingSecret The name of an existing secret with Redis® credentials\n## @param redis.architecture Redis® architecture. Allowed values: `standalone` or `replication`\n##\nredis:\n enabled: true\n auth:\n enabled: true\n ## Redis® password (both master and slave). Defaults to a random 10-character alphanumeric string if not set and auth.enabled is true.\n ## It should always be set using the password value or in the existingSecret to avoid issues\n ## with Airflow.\n ## The password value is ignored if existingSecret is set\n password: \"\"\n existingSecret: \"\"\n architecture: standalone\n\n## External Redis® configuration\n## All of these values are only used when redis.enabled is set to false\n## @param externalRedis.host Redis® host\n## @param externalRedis.port Redis® port number\n## @param externalRedis.username Redis® username\n## @param externalRedis.password Redis® password\n## @param externalRedis.existingSecret Name of an existing secret resource containing the Redis&trade credentials\n## @param externalRedis.existingSecretPasswordKey Name of an existing secret key containing the Redis&trade credentials\n##\nexternalRedis:\n host: localhost\n port: 6379\n ## Most Redis® implementations do not require a username\n ## to authenticate and it should be enough with the password\n username: \"\"\n password: \"\"\n existingSecret: \"\"\n existingSecretPasswordKey: \"\"\n", + ValuesOverrideYaml: "## @section Global parameters\n## Global Docker image parameters\n## Please, note that this will override the image parameters, including dependencies, configured to use the global value\n## Current available global Docker image parameters: imageRegistry, imagePullSecrets and storageClass\n\n## @param global.imageRegistry Global Docker image registry\n## @param global.imagePullSecrets Global Docker registry secret names as an array\n## @param global.storageClass Global StorageClass for Persistent Volume(s)\n##\nglobal:\n imageRegistry: \"\"\n ## E.g.\n ## imagePullSecrets:\n ## - myRegistryKeySecretName\n ##\n imagePullSecrets: []\n storageClass: \"\"\n\n## @section Common parameters\n\n## @param kubeVersion Override Kubernetes version\n##\nkubeVersion: \"\"\n## @param nameOverride String to partially override common.names.fullname template (will maintain the release name)\n##\nnameOverride: \"\"\n## @param fullnameOverride String to fully override common.names.fullname template\n##\nfullnameOverride: \"\"\n## @param clusterDomain Kubernetes Cluster Domain\n##\nclusterDomain: cluster.local\n## @param extraDeploy Extra objects to deploy (evaluated as a template)\n##\nextraDeploy: []\n## @param commonLabels Add labels to all the deployed resources\n##\ncommonLabels: {}\n## @param commonAnnotations Add annotations to all the deployed resources\n##\ncommonAnnotations: {}\n## Enable diagnostic mode in the deployment(s)/statefulset(s)\n##\ndiagnosticMode:\n ## @param diagnosticMode.enabled Enable diagnostic mode (all probes will be disabled and the command will be overridden)\n ##\n enabled: false\n ## @param diagnosticMode.command Command to override all containers in the the deployment(s)/statefulset(s)\n ##\n command:\n - sleep\n ## @param diagnosticMode.args Args to override all containers in the the deployment(s)/statefulset(s)\n ##\n args:\n - infinity\n\n## @section Airflow common parameters\n\n## Authentication parameters\n## ref: https://github.com/bitnami/containers/tree/main/bitnami/airflow#environment-variables\n##\nauth:\n ## @param auth.username Username to access web UI\n ##\n username: user\n ## @param auth.password Password to access web UI\n ##\n password: \"\"\n ## @param auth.fernetKey Fernet key to secure connections\n ## ref: https://airflow.readthedocs.io/en/stable/howto/secure-connections.html\n ## ref: https://bcb.github.io/airflow/fernet-key\n ##\n fernetKey: \"\"\n ## @param auth.secretKey Secret key to run your flask app\n ## ref: https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.html#secret-key\n ##\n secretKey: \"\"\n ## @param auth.existingSecret Name of an existing secret to use for Airflow credentials\n ## `auth.password`, `auth.fernetKey`, and `auth.secretKey` will be ignored and picked up from this secret\n ## The secret must contain the keys `airflow-password`, `airflow-fernet-key` and `airflow-secret-key'\n ## The value is evaluated as a template\n ##\n existingSecret: \"\"\n## @param executor Airflow executor. Allowed values: `SequentialExecutor`, `LocalExecutor`, `CeleryExecutor`, `KubernetesExecutor`, `CeleryKubernetesExecutor` and `LocalKubernetesExecutor`\n## ref: http://airflow.apache.org/docs/stable/executor/index.html\n##\nexecutor: CeleryExecutor\n## @param loadExamples Switch to load some Airflow examples\n##\nloadExamples: false\n## @param configuration Specify content for Airflow config file (auto-generated based on other env. vars otherwise)\n## e.g:\n## configuration: |-\n## [core]\n## dags_folder=/opt/bitnami/airflow/dags\n## ...\n##\nconfiguration: \"\"\n## @param existingConfigmap Name of an existing ConfigMap with the Airflow config file\n##\nexistingConfigmap: \"\"\n## Load custom DAGs from a ConfigMap\n## Note: an init container will be used to prepare the DAGs available in the ConfigMap to be consumed by Airflow containers\n##\ndags:\n ## @param dags.existingConfigmap Name of an existing ConfigMap with all the DAGs files you want to load in Airflow\n ##\n existingConfigmap: \"\"\n ## Bitnami Shell image\n ## ref: https://hub.docker.com/r/bitnami/bitnami-shell/tags/\n ## @param dags.image.registry Init container load-dags image registry\n ## @param dags.image.repository Init container load-dags image repository\n ## @param dags.image.tag Init container load-dags image tag (immutable tags are recommended)\n ## @param dags.image.digest Init container load-dags image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param dags.image.pullPolicy Init container load-dags image pull policy\n ## @param dags.image.pullSecrets Init container load-dags image pull secrets\n ##\n image:\n registry: docker.io\n repository: bitnami/bitnami-shell\n tag: 11-debian-11-r50\n digest: \"\"\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n## @param extraEnvVars Add extra environment variables for all the Airflow pods\n##\nextraEnvVars: []\n## @param extraEnvVarsCM ConfigMap with extra environment variables for all the Airflow pods\n##\nextraEnvVarsCM: \"\"\n## @param extraEnvVarsSecret Secret with extra environment variables for all the Airflow pods\n##\nextraEnvVarsSecret: \"\"\n## @param extraEnvVarsSecrets List of secrets with extra environment variables for all the Airflow pods\n##\nextraEnvVarsSecrets: []\n## @param sidecars Add additional sidecar containers to all the Airflow pods\n## Example:\n## sidecars:\n## - name: your-image-name\n## image: your-image\n## imagePullPolicy: Always\n## ports:\n## - name: portname\n## containerPort: 1234\n##\nsidecars: []\n## @param initContainers Add additional init containers to all the Airflow pods\n## Example:\n## initContainers:\n## - name: your-image-name\n## image: your-image\n## imagePullPolicy: Always\n## ports:\n## - name: portname\n## containerPort: 1234\n##\ninitContainers: []\n## @param extraVolumeMounts Optionally specify extra list of additional volumeMounts for all the Airflow pods\n##\nextraVolumeMounts: []\n## @param extraVolumes Optionally specify extra list of additional volumes for the all the Airflow pods\n##\nextraVolumes: []\n\n## @section Airflow web parameters\n\nweb:\n ## Bitnami Airflow image version\n ## ref: https://hub.docker.com/r/bitnami/airflow/tags/\n ## @param web.image.registry Airflow image registry\n ## @param web.image.repository Airflow image repository\n ## @param web.image.tag Airflow image tag (immutable tags are recommended)\n ## @param web.image.digest Airflow image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param web.image.pullPolicy Airflow image pull policy\n ## @param web.image.pullSecrets Airflow image pull secrets\n ## @param web.image.debug Enable image debug mode\n image:\n registry: docker.io\n repository: bitnami/airflow\n tag: 2.4.2-debian-11-r6\n digest: \"\"\n ## Specify a imagePullPolicy\n ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'\n ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images\n ##\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## Set to true if you would like to see extra information on logs\n ##\n debug: false\n ## @param web.baseUrl URL used to access to Airflow web ui\n ##\n baseUrl: \"\"\n ## @param web.existingConfigmap Name of an existing config map containing the Airflow web config file\n ##\n existingConfigmap: \"\"\n ## @param web.command Override default container command (useful when using custom images)\n ##\n command: []\n ## @param web.args Override default container args (useful when using custom images)\n ##\n args: []\n ## @param web.extraEnvVars Array with extra environment variables to add Airflow web pods\n ##\n extraEnvVars: []\n ## @param web.extraEnvVarsCM ConfigMap containing extra environment variables for Airflow web pods\n ##\n extraEnvVarsCM: \"\"\n ## @param web.extraEnvVarsSecret Secret containing extra environment variables (in case of sensitive data) for Airflow web pods\n ##\n extraEnvVarsSecret: \"\"\n ## @param web.extraEnvVarsSecrets List of secrets with extra environment variables for Airflow web pods\n ##\n extraEnvVarsSecrets: []\n ## @param web.containerPorts.http Airflow web HTTP container port\n ##\n containerPorts:\n http: 8080\n ## @param web.replicaCount Number of Airflow web replicas\n ##\n replicaCount: 1\n ## Configure extra options for Airflow web containers' liveness, readiness and startup probes\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes\n ## @param web.livenessProbe.enabled Enable livenessProbe on Airflow web containers\n ## @param web.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe\n ## @param web.livenessProbe.periodSeconds Period seconds for livenessProbe\n ## @param web.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe\n ## @param web.livenessProbe.failureThreshold Failure threshold for livenessProbe\n ## @param web.livenessProbe.successThreshold Success threshold for livenessProbe\n ##\n livenessProbe:\n enabled: true\n initialDelaySeconds: 180\n periodSeconds: 20\n timeoutSeconds: 5\n failureThreshold: 6\n successThreshold: 1\n ## @param web.readinessProbe.enabled Enable readinessProbe on Airflow web containers\n ## @param web.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe\n ## @param web.readinessProbe.periodSeconds Period seconds for readinessProbe\n ## @param web.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe\n ## @param web.readinessProbe.failureThreshold Failure threshold for readinessProbe\n ## @param web.readinessProbe.successThreshold Success threshold for readinessProbe\n ##\n readinessProbe:\n enabled: true\n initialDelaySeconds: 30\n periodSeconds: 10\n timeoutSeconds: 5\n failureThreshold: 6\n successThreshold: 1\n ## @param web.startupProbe.enabled Enable startupProbe on Airflow web containers\n ## @param web.startupProbe.initialDelaySeconds Initial delay seconds for startupProbe\n ## @param web.startupProbe.periodSeconds Period seconds for startupProbe\n ## @param web.startupProbe.timeoutSeconds Timeout seconds for startupProbe\n ## @param web.startupProbe.failureThreshold Failure threshold for startupProbe\n ## @param web.startupProbe.successThreshold Success threshold for startupProbe\n ##\n startupProbe:\n enabled: false\n initialDelaySeconds: 60\n periodSeconds: 10\n timeoutSeconds: 1\n failureThreshold: 15\n successThreshold: 1\n ## @param web.customLivenessProbe Custom livenessProbe that overrides the default one\n ##\n customLivenessProbe: {}\n ## @param web.customReadinessProbe Custom readinessProbe that overrides the default one\n ##\n customReadinessProbe: {}\n ## @param web.customStartupProbe Custom startupProbe that overrides the default one\n ##\n customStartupProbe: {}\n ## Airflow web resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ## @param web.resources.limits The resources limits for the Airflow web containers\n ## @param web.resources.requests The requested resources for the Airflow web containers\n ##\n resources:\n limits: {}\n requests: {}\n ## Configure Airflow web pods Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod\n ## @param web.podSecurityContext.enabled Enabled Airflow web pods' Security Context\n ## @param web.podSecurityContext.fsGroup Set Airflow web pod's Security Context fsGroup\n ##\n podSecurityContext:\n enabled: true\n fsGroup: 1001\n ## Configure Airflow web containers (only main one) Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container\n ## @param web.containerSecurityContext.enabled Enabled Airflow web containers' Security Context\n ## @param web.containerSecurityContext.runAsUser Set Airflow web containers' Security Context runAsUser\n ## @param web.containerSecurityContext.runAsNonRoot Set Airflow web containers' Security Context runAsNonRoot\n ##\n containerSecurityContext:\n enabled: true\n runAsUser: 1001\n runAsNonRoot: true\n ## @param web.lifecycleHooks for the Airflow web container(s) to automate configuration before or after startup\n ##\n lifecycleHooks: {}\n ## @param web.hostAliases Deployment pod host aliases\n ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/\n ##\n hostAliases: []\n ## @param web.podLabels Add extra labels to the Airflow web pods\n ##\n podLabels: {}\n ## @param web.podAnnotations Add extra annotations to the Airflow web pods\n ##\n podAnnotations: {}\n ## @param web.affinity Affinity for Airflow web pods assignment (evaluated as a template)\n ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity\n ## Note: `web.podAffinityPreset`, `web.podAntiAffinityPreset`, and `web.nodeAffinityPreset` will be ignored when it's set\n ##\n affinity: {}\n ## Node affinity preset\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity\n ## @param web.nodeAffinityPreset.key Node label key to match. Ignored if `web.affinity` is set.\n ## @param web.nodeAffinityPreset.type Node affinity preset type. Ignored if `web.affinity` is set. Allowed values: `soft` or `hard`\n ## @param web.nodeAffinityPreset.values Node label values to match. Ignored if `web.affinity` is set.\n ##\n nodeAffinityPreset:\n ## e.g:\n ## key: \"kubernetes.io/e2e-az-name\"\n ##\n key: \"\"\n type: \"\"\n ## e.g:\n ## values:\n ## - e2e-az1\n ## - e2e-az2\n ##\n values: []\n ## @param web.nodeSelector Node labels for Airflow web pods assignment\n ## Ref: https://kubernetes.io/docs/user-guide/node-selection/\n ##\n nodeSelector: {}\n ## @param web.podAffinityPreset Pod affinity preset. Ignored if `web.affinity` is set. Allowed values: `soft` or `hard`.\n ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAffinityPreset: \"\"\n ## @param web.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `web.affinity` is set. Allowed values: `soft` or `hard`.\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAntiAffinityPreset: soft\n ## @param web.tolerations Tolerations for Airflow web pods assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n ##\n tolerations: []\n ## @param web.topologySpreadConstraints Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template\n ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods\n ##\n topologySpreadConstraints: []\n ## @param web.priorityClassName Priority Class Name\n ## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass\n ##\n priorityClassName: \"\"\n ## @param web.schedulerName Use an alternate scheduler, e.g. \"stork\".\n ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/\n ##\n schedulerName: \"\"\n ## @param web.terminationGracePeriodSeconds Seconds Airflow web pod needs to terminate gracefully\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods\n ##\n terminationGracePeriodSeconds: \"\"\n ## @param web.updateStrategy.type Airflow web deployment strategy type\n ## @param web.updateStrategy.rollingUpdate Airflow web deployment rolling update configuration parameters\n ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy\n ##\n updateStrategy:\n type: RollingUpdate\n rollingUpdate: {}\n ## @param web.sidecars Add additional sidecar containers to the Airflow web pods\n ## Example:\n ## sidecars:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n sidecars: []\n ## @param web.initContainers Add additional init containers to the Airflow web pods\n ## Example:\n ## initContainers:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n initContainers: []\n ## @param web.extraVolumeMounts Optionally specify extra list of additional volumeMounts for the Airflow web pods\n ##\n extraVolumeMounts: []\n ## @param web.extraVolumes Optionally specify extra list of additional volumes for the Airflow web pods\n ##\n extraVolumes: []\n ## Airflow web Pod Disruption Budget\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/\n ## @param web.pdb.create Deploy a pdb object for the Airflow web pods\n ## @param web.pdb.minAvailable Maximum number/percentage of unavailable Airflow web replicas\n ## @param web.pdb.maxUnavailable Maximum number/percentage of unavailable Airflow web replicas\n ##\n pdb:\n create: false\n minAvailable: 1\n maxUnavailable: \"\"\n\n## @section Airflow scheduler parameters\n\nscheduler:\n ## Bitnami Airflow Scheduler image version\n ## ref: https://hub.docker.com/r/bitnami/airflow-scheduler/tags/\n ## @param scheduler.image.registry Airflow Scheduler image registry\n ## @param scheduler.image.repository Airflow Scheduler image repository\n ## @param scheduler.image.tag Airflow Scheduler image tag (immutable tags are recommended)\n ## @param scheduler.image.digest Airflow Schefuler image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param scheduler.image.pullPolicy Airflow Scheduler image pull policy\n ## @param scheduler.image.pullSecrets Airflow Scheduler image pull secrets\n ## @param scheduler.image.debug Enable image debug mode\n ##\n image:\n registry: docker.io\n repository: bitnami/airflow-scheduler\n tag: 2.4.2-debian-11-r4\n digest: \"\"\n ## Specify a imagePullPolicy\n ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'\n ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images\n ##\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## Set to true if you would like to see extra information on logs\n ##\n debug: false\n ## @param scheduler.replicaCount Number of scheduler replicas\n ##\n replicaCount: 1\n ## @param scheduler.command Override cmd\n ##\n command: []\n ## @param scheduler.args Override args\n ##\n args: []\n ## @param scheduler.extraEnvVars Add extra environment variables\n ##\n extraEnvVars: []\n ## @param scheduler.extraEnvVarsCM ConfigMap with extra environment variables\n ##\n extraEnvVarsCM: \"\"\n ## @param scheduler.extraEnvVarsSecret Secret with extra environment variables\n ##\n extraEnvVarsSecret: \"\"\n ## @param scheduler.extraEnvVarsSecrets List of secrets with extra environment variables for Airflow scheduler pods\n ##\n extraEnvVarsSecrets: []\n ## @param scheduler.customLivenessProbe Custom livenessProbe that overrides the default one\n ##\n customLivenessProbe: {}\n ## @param scheduler.customReadinessProbe Custom readinessProbe that overrides the default one\n ##\n customReadinessProbe: {}\n ## @param scheduler.customStartupProbe Custom startupProbe that overrides the default one\n ##\n customStartupProbe: {}\n ## Airflow scheduler resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ## @param scheduler.resources.limits The resources limits for the Airflow scheduler containers\n ## @param scheduler.resources.requests The requested resources for the Airflow scheduler containers\n ##\n resources:\n limits: {}\n requests: {}\n ## Configure Airflow scheduler pods Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod\n ## @param scheduler.podSecurityContext.enabled Enabled Airflow scheduler pods' Security Context\n ## @param scheduler.podSecurityContext.fsGroup Set Airflow scheduler pod's Security Context fsGroup\n ##\n podSecurityContext:\n enabled: true\n fsGroup: 1001\n ## Configure Airflow scheduler containers (only main one) Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container\n ## @param scheduler.containerSecurityContext.enabled Enabled Airflow scheduler containers' Security Context\n ## @param scheduler.containerSecurityContext.runAsUser Set Airflow scheduler containers' Security Context runAsUser\n ## @param scheduler.containerSecurityContext.runAsNonRoot Set Airflow scheduler containers' Security Context runAsNonRoot\n ##\n containerSecurityContext:\n enabled: true\n runAsUser: 1001\n runAsNonRoot: true\n ## @param scheduler.lifecycleHooks for the Airflow scheduler container(s) to automate configuration before or after startup\n ##\n lifecycleHooks: {}\n ## @param scheduler.hostAliases Deployment pod host aliases\n ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/\n ##\n hostAliases: []\n ## @param scheduler.podLabels Add extra labels to the Airflow scheduler pods\n ##\n podLabels: {}\n ## @param scheduler.podAnnotations Add extra annotations to the Airflow scheduler pods\n ##\n podAnnotations: {}\n ## @param scheduler.affinity Affinity for Airflow scheduler pods assignment (evaluated as a template)\n ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity\n ## Note: `scheduler.podAffinityPreset`, `scheduler.podAntiAffinityPreset`, and `scheduler.nodeAffinityPreset` will be ignored when it's set\n ##\n affinity: {}\n ## Node affinity preset\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity\n ## @param scheduler.nodeAffinityPreset.key Node label key to match. Ignored if `scheduler.affinity` is set.\n ## @param scheduler.nodeAffinityPreset.type Node affinity preset type. Ignored if `scheduler.affinity` is set. Allowed values: `soft` or `hard`\n ## @param scheduler.nodeAffinityPreset.values Node label values to match. Ignored if `scheduler.affinity` is set.\n ##\n nodeAffinityPreset:\n ## e.g:\n ## key: \"kubernetes.io/e2e-az-name\"\n ##\n key: \"\"\n type: \"\"\n ## e.g:\n ## values:\n ## - e2e-az1\n ## - e2e-az2\n ##\n values: []\n ## @param scheduler.nodeSelector Node labels for Airflow scheduler pods assignment\n ## Ref: https://kubernetes.io/docs/user-guide/node-selection/\n ##\n nodeSelector: {}\n ## @param scheduler.podAffinityPreset Pod affinity preset. Ignored if `scheduler.affinity` is set. Allowed values: `soft` or `hard`.\n ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAffinityPreset: \"\"\n ## @param scheduler.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `scheduler.affinity` is set. Allowed values: `soft` or `hard`.\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAntiAffinityPreset: soft\n ## @param scheduler.tolerations Tolerations for Airflow scheduler pods assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n ##\n tolerations: []\n ## @param scheduler.topologySpreadConstraints Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template\n ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods\n ##\n topologySpreadConstraints: []\n ## @param scheduler.priorityClassName Priority Class Name\n ## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass\n ##\n priorityClassName: \"\"\n ## @param scheduler.schedulerName Use an alternate scheduler, e.g. \"stork\".\n ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/\n ##\n schedulerName: \"\"\n ## @param scheduler.terminationGracePeriodSeconds Seconds Airflow scheduler pod needs to terminate gracefully\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods\n ##\n terminationGracePeriodSeconds: \"\"\n ## @param scheduler.updateStrategy.type Airflow scheduler deployment strategy type\n ## @param scheduler.updateStrategy.rollingUpdate Airflow scheduler deployment rolling update configuration parameters\n ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy\n ##\n updateStrategy:\n type: RollingUpdate\n rollingUpdate: {}\n ## @param scheduler.sidecars Add additional sidecar containers to the Airflow scheduler pods\n ## Example:\n ## sidecars:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n sidecars: []\n ## @param scheduler.initContainers Add additional init containers to the Airflow scheduler pods\n ## Example:\n ## initContainers:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n initContainers: []\n ## @param scheduler.extraVolumeMounts Optionally specify extra list of additional volumeMounts for the Airflow scheduler pods\n ##\n extraVolumeMounts: []\n ## @param scheduler.extraVolumes Optionally specify extra list of additional volumes for the Airflow scheduler pods\n ##\n extraVolumes: []\n ## Airflow scheduler Pod Disruption Budget\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/\n ## @param scheduler.pdb.create Deploy a pdb object for the Airflow scheduler pods\n ## @param scheduler.pdb.minAvailable Maximum number/percentage of unavailable Airflow scheduler replicas\n ## @param scheduler.pdb.maxUnavailable Maximum number/percentage of unavailable Airflow scheduler replicas\n ##\n pdb:\n create: false\n minAvailable: 1\n maxUnavailable: \"\"\n\n## @section Airflow worker parameters\n\nworker:\n ## Bitnami Airflow Worker image version\n ## ref: https://hub.docker.com/r/bitnami/airflow-worker/tags/\n ## @param worker.image.registry Airflow Worker image registry\n ## @param worker.image.repository Airflow Worker image repository\n ## @param worker.image.tag Airflow Worker image tag (immutable tags are recommended)\n ## @param worker.image.digest Airflow Worker image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param worker.image.pullPolicy Airflow Worker image pull policy\n ## @param worker.image.pullSecrets Airflow Worker image pull secrets\n ## @param worker.image.debug Enable image debug mode\n ##\n image:\n registry: docker.io\n repository: bitnami/airflow-worker\n tag: 2.4.2-debian-11-r4\n digest: \"\"\n ## Specify a imagePullPolicy\n ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'\n ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images\n ##\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## Set to true if you would like to see extra information on logs\n ##\n debug: false\n ## @param worker.command Override default container command (useful when using custom images)\n ##\n command: []\n ## @param worker.args Override default container args (useful when using custom images)\n ##\n args: []\n ## @param worker.extraEnvVars Array with extra environment variables to add Airflow worker pods\n ##\n extraEnvVars: []\n ## @param worker.extraEnvVarsCM ConfigMap containing extra environment variables for Airflow worker pods\n ##\n extraEnvVarsCM: \"\"\n ## @param worker.extraEnvVarsSecret Secret containing extra environment variables (in case of sensitive data) for Airflow worker pods\n ##\n extraEnvVarsSecret: \"\"\n ## @param worker.extraEnvVarsSecrets List of secrets with extra environment variables for Airflow worker pods\n ##\n extraEnvVarsSecrets: []\n ## @param worker.containerPorts.http Airflow worker HTTP container port\n ##\n containerPorts:\n http: 8793\n ## @param worker.replicaCount Number of Airflow worker replicas\n ##\n replicaCount: 1\n ## Configure extra options for Airflow worker containers' liveness, readiness and startup probes\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes\n ## @param worker.livenessProbe.enabled Enable livenessProbe on Airflow worker containers\n ## @param worker.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe\n ## @param worker.livenessProbe.periodSeconds Period seconds for livenessProbe\n ## @param worker.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe\n ## @param worker.livenessProbe.failureThreshold Failure threshold for livenessProbe\n ## @param worker.livenessProbe.successThreshold Success threshold for livenessProbe\n ##\n livenessProbe:\n enabled: true\n initialDelaySeconds: 180\n periodSeconds: 20\n timeoutSeconds: 5\n failureThreshold: 6\n successThreshold: 1\n ## @param worker.readinessProbe.enabled Enable readinessProbe on Airflow worker containers\n ## @param worker.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe\n ## @param worker.readinessProbe.periodSeconds Period seconds for readinessProbe\n ## @param worker.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe\n ## @param worker.readinessProbe.failureThreshold Failure threshold for readinessProbe\n ## @param worker.readinessProbe.successThreshold Success threshold for readinessProbe\n ##\n readinessProbe:\n enabled: true\n initialDelaySeconds: 30\n periodSeconds: 10\n timeoutSeconds: 5\n failureThreshold: 6\n successThreshold: 1\n ## @param worker.startupProbe.enabled Enable startupProbe on Airflow worker containers\n ## @param worker.startupProbe.initialDelaySeconds Initial delay seconds for startupProbe\n ## @param worker.startupProbe.periodSeconds Period seconds for startupProbe\n ## @param worker.startupProbe.timeoutSeconds Timeout seconds for startupProbe\n ## @param worker.startupProbe.failureThreshold Failure threshold for startupProbe\n ## @param worker.startupProbe.successThreshold Success threshold for startupProbe\n ##\n startupProbe:\n enabled: false\n initialDelaySeconds: 60\n periodSeconds: 10\n timeoutSeconds: 1\n failureThreshold: 15\n successThreshold: 1\n ## @param worker.customLivenessProbe Custom livenessProbe that overrides the default one\n ##\n customLivenessProbe: {}\n ## @param worker.customReadinessProbe Custom readinessProbe that overrides the default one\n ##\n customReadinessProbe: {}\n ## @param worker.customStartupProbe Custom startupProbe that overrides the default one\n ##\n customStartupProbe: {}\n ## Airflow worker resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ## @param worker.resources.limits The resources limits for the Airflow worker containers\n ## @param worker.resources.requests The requested resources for the Airflow worker containers\n ##\n resources:\n limits: {}\n requests: {}\n ## Configure Airflow worker pods Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod\n ## @param worker.podSecurityContext.enabled Enabled Airflow worker pods' Security Context\n ## @param worker.podSecurityContext.fsGroup Set Airflow worker pod's Security Context fsGroup\n ##\n podSecurityContext:\n enabled: true\n fsGroup: 1001\n ## Configure Airflow worker containers (only main one) Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container\n ## @param worker.containerSecurityContext.enabled Enabled Airflow worker containers' Security Context\n ## @param worker.containerSecurityContext.runAsUser Set Airflow worker containers' Security Context runAsUser\n ## @param worker.containerSecurityContext.runAsNonRoot Set Airflow worker containers' Security Context runAsNonRoot\n ##\n containerSecurityContext:\n enabled: true\n runAsUser: 1001\n runAsNonRoot: true\n ## @param worker.lifecycleHooks for the Airflow worker container(s) to automate configuration before or after startup\n ##\n lifecycleHooks: {}\n ## @param worker.hostAliases Deployment pod host aliases\n ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/\n ##\n hostAliases: []\n ## @param worker.podLabels Add extra labels to the Airflow worker pods\n ##\n podLabels: {}\n ## @param worker.podAnnotations Add extra annotations to the Airflow worker pods\n ##\n podAnnotations: {}\n ## @param worker.affinity Affinity for Airflow worker pods assignment (evaluated as a template)\n ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity\n ## Note: `worker.podAffinityPreset`, `worker.podAntiAffinityPreset`, and `worker.nodeAffinityPreset` will be ignored when it's set\n ##\n affinity: {}\n ## Node affinity preset\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity\n ## @param worker.nodeAffinityPreset.key Node label key to match. Ignored if `worker.affinity` is set.\n ## @param worker.nodeAffinityPreset.type Node affinity preset type. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard`\n ## @param worker.nodeAffinityPreset.values Node label values to match. Ignored if `worker.affinity` is set.\n ##\n nodeAffinityPreset:\n ## e.g:\n ## key: \"kubernetes.io/e2e-az-name\"\n ##\n key: \"\"\n type: \"\"\n ## e.g:\n ## values:\n ## - e2e-az1\n ## - e2e-az2\n ##\n values: []\n ## @param worker.nodeSelector Node labels for Airflow worker pods assignment\n ## Ref: https://kubernetes.io/docs/user-guide/node-selection/\n ##\n nodeSelector: {}\n ## @param worker.podAffinityPreset Pod affinity preset. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard`.\n ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAffinityPreset: \"\"\n ## @param worker.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard`.\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAntiAffinityPreset: soft\n ## @param worker.tolerations Tolerations for Airflow worker pods assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n ##\n tolerations: []\n ## @param worker.topologySpreadConstraints Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template\n ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods\n ##\n topologySpreadConstraints: []\n ## @param worker.priorityClassName Priority Class Name\n ## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass\n ##\n priorityClassName: \"\"\n ## @param worker.schedulerName Use an alternate scheduler, e.g. \"stork\".\n ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/\n ##\n schedulerName: \"\"\n ## @param worker.terminationGracePeriodSeconds Seconds Airflow worker pod needs to terminate gracefully\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods\n ##\n terminationGracePeriodSeconds: \"\"\n ## @param worker.updateStrategy.type Airflow worker deployment strategy type\n ## @param worker.updateStrategy.rollingUpdate Airflow worker deployment rolling update configuration parameters\n ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy\n ##\n updateStrategy:\n type: RollingUpdate\n rollingUpdate: {}\n ## @param worker.sidecars Add additional sidecar containers to the Airflow worker pods\n ## Example:\n ## sidecars:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n sidecars: []\n ## @param worker.initContainers Add additional init containers to the Airflow worker pods\n ## Example:\n ## initContainers:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n initContainers: []\n ## @param worker.extraVolumeMounts Optionally specify extra list of additional volumeMounts for the Airflow worker pods\n ##\n extraVolumeMounts: []\n ## @param worker.extraVolumes Optionally specify extra list of additional volumes for the Airflow worker pods\n ##\n extraVolumes: []\n ## @param worker.extraVolumeClaimTemplates Optionally specify extra list of volumesClaimTemplates for the Airflow worker statefulset\n ##\n extraVolumeClaimTemplates: []\n ## @param worker.podTemplate Template to replace the default one to be use when `executor=KubernetesExecutor` to create Airflow worker pods\n ##\n podTemplate: {}\n ## Airflow worker Pod Disruption Budget\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/\n ## @param worker.pdb.create Deploy a pdb object for the Airflow worker pods\n ## @param worker.pdb.minAvailable Maximum number/percentage of unavailable Airflow worker replicas\n ## @param worker.pdb.maxUnavailable Maximum number/percentage of unavailable Airflow worker replicas\n ##\n pdb:\n create: false\n minAvailable: 1\n maxUnavailable: \"\"\n ## Enable HorizontalPodAutoscaler for worker pods\n ## ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/\n ## @param worker.autoscaling.enabled Whether enable horizontal pod autoscaler\n ## @param worker.autoscaling.minReplicas Configure a minimum amount of pods\n ## @param worker.autoscaling.maxReplicas Configure a maximum amount of pods\n ## @param worker.autoscaling.targetCPU Define the CPU target to trigger the scaling actions (utilization percentage)\n ## @param worker.autoscaling.targetMemory Define the memory target to trigger the scaling actions (utilization percentage)\n ##\n autoscaling:\n enabled: false\n minReplicas: 1\n maxReplicas: 3\n targetCPU: 80\n targetMemory: 80\n\n## @section Airflow git sync parameters\n\n## Configure Git to pull dags and plugins\n##\ngit:\n ## Bitnami Git image version\n ## ref: https://hub.docker.com/r/bitnami/git/tags/\n ## @param git.image.registry Git image registry\n ## @param git.image.repository Git image repository\n ## @param git.image.tag Git image tag (immutable tags are recommended)\n ## @param git.image.digest Git image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param git.image.pullPolicy Git image pull policy\n ## @param git.image.pullSecrets Git image pull secrets\n ##\n image:\n registry: docker.io\n repository: bitnami/git\n tag: 2.38.1-debian-11-r7\n digest: \"\"\n ## Specify a imagePullPolicy\n ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'\n ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images\n ##\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## Get DAG files from git repositories\n ## @param git.dags.enabled Enable in order to download DAG files from git repositories.\n ## @param git.dags.repositories [array] Array of repositories from which to download DAG files\n ##\n dags:\n enabled: false\n ## Name for repositories can be anything unique and must follow same naming conventions as kubernetes.\n ## Kubernetes resources can have names up to 253 characters long. The characters allowed in names are:\n ## digits (0-9), lower case letters (a-z), -, and .\n ## Example:\n ## - repository: https://github.com/myuser/myrepo\n ## branch: main\n ## name: my-dags\n ## path: /\n ##\n repositories:\n - repository: \"\"\n ## Branch from repository to checkout\n ##\n branch: \"\"\n ## An unique identifier for repository, must be unique for each repository\n ##\n name: \"\"\n ## Path to a folder in the repository containing the dags\n ##\n path: \"\"\n ## Get Plugins files from git repositories.\n ## @param git.plugins.enabled Enable in order to download Plugins files from git repositories.\n ## @param git.plugins.repositories [array] Array of repositories from which to download DAG files\n ##\n plugins:\n enabled: false\n repositories:\n - repository: \"\"\n ## Branch from repository to checkout\n ##\n branch: \"\"\n ## An unique identifier for repository, must be unique for each repository\n ##\n name: \"\"\n ## Path to a folder in the repository containing the plugins\n ##\n path: \"\"\n ## Properties for the Clone init container\n ## @param git.clone.command Override cmd\n ## @param git.clone.args Override args\n ## @param git.clone.extraVolumeMounts Add extra volume mounts\n ## @param git.clone.extraEnvVars Add extra environment variables\n ## @param git.clone.extraEnvVarsCM ConfigMap with extra environment variables\n ## @param git.clone.extraEnvVarsSecret Secret with extra environment variables\n ## @param git.clone.resources Clone init container resource requests and limits\n ##\n clone:\n command: []\n args: []\n extraVolumeMounts: []\n extraEnvVars: []\n extraEnvVarsCM: \"\"\n extraEnvVarsSecret: \"\"\n ## Clone init container resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ##\n resources: {}\n ## Properties for the Sync sidecar container\n ## @param git.sync.interval Interval in seconds to pull the git repository containing the plugins and/or DAG files\n ## @param git.sync.command Override cmd\n ## @param git.sync.args Override args\n ## @param git.sync.extraVolumeMounts Add extra volume mounts\n ## @param git.sync.extraEnvVars Add extra environment variables\n ## @param git.sync.extraEnvVarsCM ConfigMap with extra environment variables\n ## @param git.sync.extraEnvVarsSecret Secret with extra environment variables\n ## @param git.sync.resources Sync sidecar container resource requests and limits\n ##\n sync:\n interval: 60\n command: []\n args: []\n extraVolumeMounts: []\n extraEnvVars: []\n extraEnvVarsCM: \"\"\n extraEnvVarsSecret: \"\"\n ## Sync sidecar container resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ##\n resources: {}\n\n## @section Airflow ldap parameters\n\n## LDAP configuration\n## @param ldap.enabled Enable LDAP authentication\n## @param ldap.uri Server URI, eg. ldap://ldap_server:389\n## DEPRECATED ldap.base It will be removed in a future release, please use 'ldap.basedn' instead\n## @param ldap.basedn Base of the search, eg. ou=example,o=org.\n## DEPRECATED ldap.uidField It will be removed in a future release,, please use 'ldap.searchAttribute' instead\n## @param ldap.searchAttribute if doing an indirect bind to ldap, this is the field that matches the username when searching for the account to bind to\n## @param ldap.binddn DN of the account used to search in the LDAP server.\n## @param ldap.bindpw Bind Password\n## @param ldap.userRegistration Set to True to enable user self registration\n## @param ldap.userRegistrationRole Set role name to be assign when a user registers himself. This role must already exist. Mandatory when using ldap.userRegistration\n## @param ldap.rolesMapping mapping from LDAP DN to a list of roles\n## @param ldap.rolesSyncAtLogin replace ALL the user's roles each login, or only on registration\n##\nldap:\n enabled: false\n uri: \"ldap://ldap_server:389\"\n basedn: \"dc=example,dc=org\"\n searchAttribute: \"cn\"\n binddn: \"cn=admin,dc=example,dc=org\"\n bindpw: \"\"\n userRegistration: 'True'\n userRegistrationRole: \"Public\"\n rolesMapping: '{ \"cn=All,ou=Groups,dc=example,dc=org\": [\"User\"], \"cn=Admins,ou=Groups,dc=example,dc=org\": [\"Admin\"], }'\n rolesSyncAtLogin: 'True'\n\n ## SSL/TLS parameters for LDAP\n ## @param ldap.tls.enabled Enabled TLS/SSL for LDAP, you must include the CA file.\n ## @param ldap.tls.allowSelfSigned Allow to use self signed certificates\n ## DEPRECATED ldap.tls.CAcertificateSecret It will be removed in a future release, please use ldap.tls.certificatesSecret instead\n ## @param ldap.tls.certificatesSecret Name of the existing secret containing the certificate CA file that will be used by ldap client\n ## @param ldap.tls.certificatesMountPath Where LDAP certifcates are mounted.\n ## DEPRECATED ldap.tls.CAcertificateFilename It will be removed in a future release, please use ldap.tls.CAFilename instead\n ## @param ldap.tls.CAFilename LDAP CA cert filename\n ##\n tls:\n enabled: false\n allowSelfSigned: true\n certificatesSecret: \"\"\n certificatesMountPath: /opt/bitnami/airflow/conf/certs\n CAFilename: \"\"\n\n## @section Traffic Exposure Parameters\n\n## Airflow service parameters\n##\nservice:\n ## @param service.type Airflow service type\n ##\n type: ClusterIP\n ## @param service.ports.http Airflow service HTTP port\n ##\n ports:\n http: 8080\n ## Node ports to expose\n ## @param service.nodePorts.http Node port for HTTP\n ## NOTE: choose port between <30000-32767>\n ##\n nodePorts:\n http: \"\"\n ## @param service.sessionAffinity Control where client requests go, to the same pod or round-robin\n ## Values: ClientIP or None\n ## ref: https://kubernetes.io/docs/user-guide/services/\n ##\n sessionAffinity: None\n ## @param service.sessionAffinityConfig Additional settings for the sessionAffinity\n ## sessionAffinityConfig:\n ## clientIP:\n ## timeoutSeconds: 300\n ##\n sessionAffinityConfig: {}\n ## @param service.clusterIP Airflow service Cluster IP\n ## e.g.:\n ## clusterIP: None\n ##\n clusterIP: \"\"\n ## @param service.loadBalancerIP Airflow service Load Balancer IP\n ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-loadbalancer\n ##\n loadBalancerIP: \"\"\n ## @param service.loadBalancerSourceRanges Airflow service Load Balancer sources\n ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service\n ## e.g:\n ## loadBalancerSourceRanges:\n ## - 10.10.10.0/24\n ##\n loadBalancerSourceRanges: []\n ## @param service.externalTrafficPolicy Airflow service external traffic policy\n ## ref https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip\n ##\n externalTrafficPolicy: Cluster\n ## @param service.annotations Additional custom annotations for Airflow service\n ##\n annotations: {}\n ## @param service.extraPorts Extra port to expose on Airflow service\n ##\n extraPorts: []\n\n## Airflow ingress parameters\n## ref: https://kubernetes.io/docs/user-guide/ingress/\n##\ningress:\n ## @param ingress.enabled Enable ingress record generation for Airflow\n ##\n enabled: false\n ## @param ingress.ingressClassName IngressClass that will be be used to implement the Ingress (Kubernetes 1.18+)\n ## This is supported in Kubernetes 1.18+ and required if you have more than one IngressClass marked as the default for your cluster .\n ## ref: https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/\n ##\n ingressClassName: \"\"\n ## @param ingress.pathType Ingress path type\n ##\n pathType: ImplementationSpecific\n ## @param ingress.apiVersion Force Ingress API version (automatically detected if not set)\n ##\n apiVersion: \"\"\n ## @param ingress.hostname Default host for the ingress record\n ##\n hostname: airflow.local\n ## @param ingress.path Default path for the ingress record\n ## NOTE: You may need to set this to '/*' in order to use this with ALB ingress controllers\n ##\n path: /\n ## @param ingress.annotations [object] Additional annotations for the Ingress resource. To enable certificate autogeneration, place here your cert-manager annotations.\n ## Use this parameter to set the required annotations for cert-manager, see\n ## ref: https://cert-manager.io/docs/usage/ingress/#supported-annotations\n ## e.g:\n ## annotations:\n ## kubernetes.io/ingress.class: nginx\n ## cert-manager.io/cluster-issuer: cluster-issuer-name\n ##\n annotations: {}\n ## @param ingress.tls Enable TLS configuration for the host defined at `ingress.hostname` parameter\n ## TLS certificates will be retrieved from a TLS secret with name: `{{- printf \"%s-tls\" .Values.ingress.hostname }}`\n ## You can:\n ## - Use the `ingress.secrets` parameter to create this TLS secret\n ## - Rely on cert-manager to create it by setting the corresponding annotations\n ## - Rely on Helm to create self-signed certificates by setting `ingress.selfSigned=true`\n ##\n tls: false\n ## @param ingress.selfSigned Create a TLS secret for this ingress record using self-signed certificates generated by Helm\n ##\n selfSigned: false\n ## @param ingress.extraHosts An array with additional hostname(s) to be covered with the ingress record\n ## e.g:\n ## extraHosts:\n ## - name: airflow.local\n ## path: /\n ##\n extraHosts: []\n ## @param ingress.extraPaths An array with additional arbitrary paths that may need to be added to the ingress under the main host\n ## e.g:\n ## extraPaths:\n ## - path: /*\n ## backend:\n ## serviceName: ssl-redirect\n ## servicePort: use-annotation\n ##\n extraPaths: []\n ## @param ingress.extraTls TLS configuration for additional hostname(s) to be covered with this ingress record\n ## ref: https://kubernetes.io/docs/concepts/services-networking/ingress/#tls\n ## e.g:\n ## extraTls:\n ## - hosts:\n ## - airflow.local\n ## secretName: airflow.local-tls\n ##\n extraTls: []\n ## @param ingress.secrets Custom TLS certificates as secrets\n ## NOTE: 'key' and 'certificate' are expected in PEM format\n ## NOTE: 'name' should line up with a 'secretName' set further up\n ## If it is not set and you're using cert-manager, this is unneeded, as it will create a secret for you with valid certificates\n ## If it is not set and you're NOT using cert-manager either, self-signed certificates will be created valid for 365 days\n ## It is also possible to create and manage the certificates outside of this helm chart\n ## Please see README.md for more information\n ## e.g:\n ## secrets:\n ## - name: airflow.local-tls\n ## key: |-\n ## -----BEGIN RSA PRIVATE KEY-----\n ## ...\n ## -----END RSA PRIVATE KEY-----\n ## certificate: |-\n ## -----BEGIN CERTIFICATE-----\n ## ...\n ## -----END CERTIFICATE-----\n ##\n secrets: []\n ## @param ingress.extraRules Additional rules to be covered with this ingress record\n ## ref: https://kubernetes.io/docs/concepts/services-networking/ingress/#ingress-rules\n ## e.g:\n ## extraRules:\n ## - host: example.local\n ## http:\n ## path: /\n ## backend:\n ## service:\n ## name: example-svc\n ## port:\n ## name: http\n ##\n extraRules: []\n\n## @section Other Parameters\n\n## Service account for Airflow pods to use.\n## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/\n##\nserviceAccount:\n ## @param serviceAccount.create Enable creation of ServiceAccount for Airflow pods\n ##\n create: false\n ## @param serviceAccount.name The name of the ServiceAccount to use.\n ## If not set and create is true, a name is generated using the common.names.fullname template\n ##\n name: \"\"\n ## @param serviceAccount.automountServiceAccountToken Allows auto mount of ServiceAccountToken on the serviceAccount created\n ## Can be set to false if pods using this serviceAccount do not need to use K8s API\n ##\n automountServiceAccountToken: true\n ## @param serviceAccount.annotations Additional custom annotations for the ServiceAccount\n ##\n annotations: {}\n## Role Based Access\n## Ref: https://kubernetes.io/docs/admin/authorization/rbac/\n## @param rbac.create Create Role and RoleBinding\n##\nrbac:\n create: false\n ## @param rbac.rules Custom RBAC rules to set\n ## e.g:\n ## rules:\n ## - apiGroups:\n ## - \"\"\n ## resources:\n ## - pods\n ## verbs:\n ## - get\n ## - list\n ##\n rules: []\n\n## @section Airflow metrics parameters\n\nmetrics:\n ## @param metrics.enabled Whether or not to create a standalone Airflow exporter to expose Airflow metrics\n ##\n enabled: false\n ## Bitnami Airflow exporter image\n ## ref: https://hub.docker.com/r/bitnami/airflow-exporter/tags/\n ## @param metrics.image.registry Airflow exporter image registry\n ## @param metrics.image.repository Airflow exporter image repository\n ## @param metrics.image.tag Airflow exporter image tag (immutable tags are recommended)\n ## @param metrics.image.digest Airflow exporter image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param metrics.image.pullPolicy Airflow exporter image pull policy\n ## @param metrics.image.pullSecrets Airflow exporter image pull secrets\n ##\n image:\n registry: docker.io\n repository: bitnami/airflow-exporter\n tag: 0.20220314.0-debian-11-r57\n digest: \"\"\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## @param metrics.extraEnvVars Array with extra environment variables to add Airflow exporter pods\n ##\n extraEnvVars: []\n ## @param metrics.extraEnvVarsCM ConfigMap containing extra environment variables for Airflow exporter pods\n ##\n extraEnvVarsCM: \"\"\n ## @param metrics.extraEnvVarsSecret Secret containing extra environment variables (in case of sensitive data) for Airflow exporter pods\n ##\n extraEnvVarsSecret: \"\"\n ## @param metrics.containerPorts.http Airflow exporter metrics container port\n ##\n containerPorts:\n http: 9112\n ## Airflow exporter resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ## @param metrics.resources.limits The resources limits for the container\n ## @param metrics.resources.requests The requested resources for the container\n ##\n resources:\n limits: {}\n requests: {}\n ## Airflow exporter pods' Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod\n ## @param metrics.podSecurityContext.enabled Enable security context for the pods\n ## @param metrics.podSecurityContext.fsGroup Set Airflow exporter pod's Security Context fsGroup\n ##\n podSecurityContext:\n enabled: true\n fsGroup: 1001\n ## Airflow exporter containers' Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container\n ## @param metrics.containerSecurityContext.enabled Enable Airflow exporter containers' Security Context\n ## @param metrics.containerSecurityContext.runAsUser Set Airflow exporter containers' Security Context runAsUser\n ## @param metrics.containerSecurityContext.runAsNonRoot Set Airflow exporter containers' Security Context runAsNonRoot\n ## e.g:\n ## containerSecurityContext:\n ## enabled: true\n ## capabilities:\n ## drop: [\"NET_RAW\"]\n ## readOnlyRootFilesystem: true\n ##\n containerSecurityContext:\n enabled: true\n runAsUser: 1001\n runAsNonRoot: true\n ## @param metrics.lifecycleHooks for the Airflow exporter container(s) to automate configuration before or after startup\n ##\n lifecycleHooks: {}\n ## @param metrics.hostAliases Airflow exporter pods host aliases\n ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/\n ##\n hostAliases: []\n ## @param metrics.podLabels Extra labels for Airflow exporter pods\n ## Ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/\n ##\n podLabels: {}\n ## @param metrics.podAnnotations Extra annotations for Airflow exporter pods\n ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/\n ##\n podAnnotations: {}\n ## @param metrics.podAffinityPreset Pod affinity preset. Ignored if `metrics.affinity` is set. Allowed values: `soft` or `hard`\n ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAffinityPreset: \"\"\n ## @param metrics.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `metrics.affinity` is set. Allowed values: `soft` or `hard`\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAntiAffinityPreset: soft\n ## Node metrics.affinity preset\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity\n ##\n nodeAffinityPreset:\n ## @param metrics.nodeAffinityPreset.type Node affinity preset type. Ignored if `metrics.affinity` is set. Allowed values: `soft` or `hard`\n ##\n type: \"\"\n ## @param metrics.nodeAffinityPreset.key Node label key to match Ignored if `metrics.affinity` is set.\n ## E.g.\n ## key: \"kubernetes.io/e2e-az-name\"\n ##\n key: \"\"\n ## @param metrics.nodeAffinityPreset.values Node label values to match. Ignored if `metrics.affinity` is set.\n ## E.g.\n ## values:\n ## - e2e-az1\n ## - e2e-az2\n ##\n values: []\n ## @param metrics.affinity Affinity for pod assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity\n ## Note: metrics.podAffinityPreset, metrics.podAntiAffinityPreset, and metrics.nodeAffinityPreset will be ignored when it's set\n ##\n affinity: {}\n ## @param metrics.nodeSelector Node labels for pod assignment\n ## Ref: https://kubernetes.io/docs/user-guide/node-selection/\n ##\n nodeSelector: {}\n ## @param metrics.tolerations Tolerations for pod assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n ##\n tolerations: []\n ## @param metrics.schedulerName Name of the k8s scheduler (other than default) for Airflow exporter\n ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/\n ##\n schedulerName: \"\"\n ## Airflow exporter service configuration\n ##\n service:\n ## @param metrics.service.ports.http Airflow exporter metrics service port\n ##\n ports:\n http: 9112\n ## @param metrics.service.clusterIP Static clusterIP or None for headless services\n ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#choosing-your-own-ip-address\n ##\n clusterIP: \"\"\n ## @param metrics.service.sessionAffinity Control where client requests go, to the same pod or round-robin\n ## Values: ClientIP or None\n ## ref: https://kubernetes.io/docs/user-guide/services/\n ##\n sessionAffinity: None\n ## @param metrics.service.annotations [object] Annotations for the Airflow exporter service\n ##\n annotations:\n prometheus.io/scrape: \"true\"\n prometheus.io/port: \"{{ .Values.metrics.service.ports.http }}\"\n ## Prometheus Operator ServiceMonitor configuration\n ##\n serviceMonitor:\n ## @param metrics.serviceMonitor.enabled if `true`, creates a Prometheus Operator ServiceMonitor (requires `metrics.enabled` to be `true`)\n ##\n enabled: false\n ## @param metrics.serviceMonitor.namespace Namespace in which Prometheus is running\n ##\n namespace: \"\"\n ## @param metrics.serviceMonitor.interval Interval at which metrics should be scraped\n ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint\n ##\n interval: \"\"\n ## @param metrics.serviceMonitor.scrapeTimeout Timeout after which the scrape is ended\n ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint\n ##\n scrapeTimeout: \"\"\n ## @param metrics.serviceMonitor.labels Additional labels that can be used so ServiceMonitor will be discovered by Prometheus\n ##\n labels: {}\n ## @param metrics.serviceMonitor.selector Prometheus instance selector labels\n ## ref: https://github.com/bitnami/charts/tree/main/bitnami/prometheus-operator#prometheus-configuration\n ##\n selector: {}\n ## @param metrics.serviceMonitor.relabelings RelabelConfigs to apply to samples before scraping\n ##\n relabelings: []\n ## @param metrics.serviceMonitor.metricRelabelings MetricRelabelConfigs to apply to samples before ingestion\n ##\n metricRelabelings: []\n ## @param metrics.serviceMonitor.honorLabels Specify honorLabels parameter to add the scrape endpoint\n ##\n honorLabels: false\n ## @param metrics.serviceMonitor.jobLabel The name of the label on the target service to use as the job name in prometheus.\n ##\n jobLabel: \"\"\n\n## @section Airflow database parameters\n\n## PostgreSQL chart configuration\n## ref: https://github.com/bitnami/charts/blob/main/bitnami/postgresql/values.yaml\n## @param postgresql.enabled Switch to enable or disable the PostgreSQL helm chart\n## @param postgresql.auth.enablePostgresUser Assign a password to the \"postgres\" admin user. Otherwise, git access will be blocked for this user\n## @param postgresql.auth.username Name for a custom user to create\n## @param postgresql.auth.password Password for the custom user to create\n## @param postgresql.auth.database Name for a custom database to create\n## @param postgresql.auth.existingSecret Name of existing secret to use for PostgreSQL credentials\n## @param postgresql.architecture PostgreSQL architecture (`standalone` or `replication`)\n##\npostgresql:\n enabled: true\n auth:\n enablePostgresUser: false\n username: bn_airflow\n password: \"\"\n database: bitnami_airflow\n existingSecret: \"\"\n architecture: standalone\n## External PostgreSQL configuration\n## All of these values are only used when postgresql.enabled is set to false\n## @param externalDatabase.host Database host\n## @param externalDatabase.port Database port number\n## @param externalDatabase.user Non-root username for Airflow\n## @param externalDatabase.password Password for the non-root username for Airflow\n## @param externalDatabase.database Airflow database name\n## @param externalDatabase.existingSecret Name of an existing secret resource containing the database credentials\n## @param externalDatabase.existingSecretPasswordKey Name of an existing secret key containing the database credentials\n##\nexternalDatabase:\n host: localhost\n port: 5432\n user: bn_airflow\n database: bitnami_airflow\n password: \"\"\n existingSecret: \"\"\n existingSecretPasswordKey: \"\"\n\n## Redis® chart configuration\n## ref: https://github.com/bitnami/charts/blob/main/bitnami/redis/values.yaml\n## @param redis.enabled Switch to enable or disable the Redis® helm\n## @param redis.auth.enabled Enable password authentication\n## @param redis.auth.password Redis® password\n## @param redis.auth.existingSecret The name of an existing secret with Redis® credentials\n## @param redis.architecture Redis® architecture. Allowed values: `standalone` or `replication`\n##\nredis:\n enabled: true\n auth:\n enabled: true\n ## Redis® password (both master and slave). Defaults to a random 10-character alphanumeric string if not set and auth.enabled is true.\n ## It should always be set using the password value or in the existingSecret to avoid issues\n ## with Airflow.\n ## The password value is ignored if existingSecret is set\n password: \"\"\n existingSecret: \"\"\n architecture: standalone\n\n## External Redis® configuration\n## All of these values are only used when redis.enabled is set to false\n## @param externalRedis.host Redis® host\n## @param externalRedis.port Redis® port number\n## @param externalRedis.username Redis® username\n## @param externalRedis.password Redis® password\n## @param externalRedis.existingSecret Name of an existing secret resource containing the Redis&trade credentials\n## @param externalRedis.existingSecretPasswordKey Name of an existing secret key containing the Redis&trade credentials\n##\nexternalRedis:\n host: localhost\n port: 6379\n ## Most Redis® implementations do not require a username\n ## to authenticate and it should be enough with the password\n username: \"\"\n password: \"\"\n existingSecret: \"\"\n existingSecretPasswordKey: \"\"\n", Readme: "", UserId: 6, ReferenceValueId: 6304, @@ -93,7 +93,7 @@ func TestAppStoreDeploymentService(t *testing.T) { InstalledAppId: 0, InstalledAppVersionId: 0, AppStoreVersion: 6304, - ValuesOverrideYaml: "## @section Global parameters\n## Global Docker image parameters\n## Please, note that this will override the image parameters, including dependencies, configured to use the global value\n## Current available global Docker image parameters: imageRegistry, imagePullSecrets and storageClass\n\n## @param global.imageRegistry Global Docker image registry\n## @param global.imagePullSecrets Global Docker registry secret names as an array\n## @param global.storageClass Global StorageClass for Persistent Volume(s)\n##\nglobal:\n imageRegistry: \"\"\n ## E.g.\n ## imagePullSecrets:\n ## - myRegistryKeySecretName\n ##\n imagePullSecrets: []\n storageClass: \"\"\n\n## @section Common parameters\n\n## @param kubeVersion Override Kubernetes version\n##\nkubeVersion: \"\"\n## @param nameOverride String to partially override common.names.fullname template (will maintain the release name)\n##\nnameOverride: \"\"\n## @param fullnameOverride String to fully override common.names.fullname template\n##\nfullnameOverride: \"\"\n## @param clusterDomain Kubernetes Cluster Domain\n##\nclusterDomain: cluster.local\n## @param extraDeploy Extra objects to deploy (evaluated as a template)\n##\nextraDeploy: []\n## @param commonLabels Add labels to all the deployed resources\n##\ncommonLabels: {}\n## @param commonAnnotations Add annotations to all the deployed resources\n##\ncommonAnnotations: {}\n## Enable diagnostic mode in the deployment(s)/statefulset(s)\n##\ndiagnosticMode:\n ## @param diagnosticMode.enabled Enable diagnostic mode (all probes will be disabled and the command will be overridden)\n ##\n enabled: false\n ## @param diagnosticMode.command Command to override all containers in the the deployment(s)/statefulset(s)\n ##\n command:\n - sleep\n ## @param diagnosticMode.args Args to override all containers in the the deployment(s)/statefulset(s)\n ##\n args:\n - infinity\n\n## @section Airflow common parameters\n\n## Authentication parameters\n## ref: https://github.com/bitnami/containers/tree/main/bitnami/airflow#environment-variables\n##\nauth:\n ## @param auth.username Username to access web UI\n ##\n username: user\n ## @param auth.password Password to access web UI\n ##\n password: \"\"\n ## @param auth.fernetKey Fernet key to secure connections\n ## ref: https://airflow.readthedocs.io/en/stable/howto/secure-connections.html\n ## ref: https://bcb.github.io/airflow/fernet-key\n ##\n fernetKey: \"\"\n ## @param auth.secretKey Secret key to run your flask app\n ## ref: https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.html#secret-key\n ##\n secretKey: \"\"\n ## @param auth.existingSecret Name of an existing secret to use for Airflow credentials\n ## `auth.password`, `auth.fernetKey`, and `auth.secretKey` will be ignored and picked up from this secret\n ## The secret must contain the keys `airflow-password`, `airflow-fernet-key` and `airflow-secret-key'\n ## The value is evaluated as a template\n ##\n existingSecret: \"\"\n## @param executor Airflow executor. Allowed values: `SequentialExecutor`, `LocalExecutor`, `CeleryExecutor`, `KubernetesExecutor`, `CeleryKubernetesExecutor` and `LocalKubernetesExecutor`\n## ref: http://airflow.apache.org/docs/stable/executor/index.html\n##\nexecutor: CeleryExecutor\n## @param loadExamples Switch to load some Airflow examples\n##\nloadExamples: false\n## @param configuration Specify content for Airflow config file (auto-generated based on other env. vars otherwise)\n## e.g:\n## configuration: |-\n## [core]\n## dags_folder=/opt/bitnami/airflow/dags\n## ...\n##\nconfiguration: \"\"\n## @param existingConfigmap Name of an existing ConfigMap with the Airflow config file\n##\nexistingConfigmap: \"\"\n## Load custom DAGs from a ConfigMap\n## Note: an init container will be used to prepare the DAGs available in the ConfigMap to be consumed by Airflow containers\n##\ndags:\n ## @param dags.existingConfigmap Name of an existing ConfigMap with all the DAGs files you want to load in Airflow\n ##\n existingConfigmap: \"\"\n ## Bitnami Shell image\n ## ref: https://hub.docker.com/r/bitnami/bitnami-shell/tags/\n ## @param dags.image.registry Init container load-dags image registry\n ## @param dags.image.repository Init container load-dags image repository\n ## @param dags.image.tag Init container load-dags image tag (immutable tags are recommended)\n ## @param dags.image.digest Init container load-dags image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param dags.image.pullPolicy Init container load-dags image pull policy\n ## @param dags.image.pullSecrets Init container load-dags image pull secrets\n ##\n image:\n registry: docker.io\n repository: bitnami/bitnami-shell\n tag: 11-debian-11-r50\n digest: \"\"\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n## @param extraEnvVars Add extra environment variables for all the Airflow pods\n##\nextraEnvVars: []\n## @param extraEnvVarsCM ConfigMap with extra environment variables for all the Airflow pods\n##\nextraEnvVarsCM: \"\"\n## @param extraEnvVarsSecret Secret with extra environment variables for all the Airflow pods\n##\nextraEnvVarsSecret: \"\"\n## @param extraEnvVarsSecrets List of secrets with extra environment variables for all the Airflow pods\n##\nextraEnvVarsSecrets: []\n## @param sidecars Add additional sidecar containers to all the Airflow pods\n## Example:\n## sidecars:\n## - name: your-image-name\n## image: your-image\n## imagePullPolicy: Always\n## ports:\n## - name: portname\n## containerPort: 1234\n##\nsidecars: []\n## @param initContainers Add additional init containers to all the Airflow pods\n## Example:\n## initContainers:\n## - name: your-image-name\n## image: your-image\n## imagePullPolicy: Always\n## ports:\n## - name: portname\n## containerPort: 1234\n##\ninitContainers: []\n## @param extraVolumeMounts Optionally specify extra list of additional volumeMounts for all the Airflow pods\n##\nextraVolumeMounts: []\n## @param extraVolumes Optionally specify extra list of additional volumes for the all the Airflow pods\n##\nextraVolumes: []\n\n## @section Airflow web parameters\n\nweb:\n ## Bitnami Airflow image version\n ## ref: https://hub.docker.com/r/bitnami/airflow/tags/\n ## @param web.image.registry Airflow image registry\n ## @param web.image.repository Airflow image repository\n ## @param web.image.tag Airflow image tag (immutable tags are recommended)\n ## @param web.image.digest Airflow image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param web.image.pullPolicy Airflow image pull policy\n ## @param web.image.pullSecrets Airflow image pull secrets\n ## @param web.image.debug Enable image debug mode\n image:\n registry: docker.io\n repository: bitnami/airflow\n tag: 2.4.2-debian-11-r6\n digest: \"\"\n ## Specify a imagePullPolicy\n ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'\n ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images\n ##\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## Set to true if you would like to see extra information on logs\n ##\n debug: false\n ## @param web.baseUrl URL used to access to Airflow web ui\n ##\n baseUrl: \"\"\n ## @param web.existingConfigmap Name of an existing config map containing the Airflow web config file\n ##\n existingConfigmap: \"\"\n ## @param web.command Override default container command (useful when using custom images)\n ##\n command: []\n ## @param web.args Override default container args (useful when using custom images)\n ##\n args: []\n ## @param web.extraEnvVars Array with extra environment variables to add Airflow web pods\n ##\n extraEnvVars: []\n ## @param web.extraEnvVarsCM ConfigMap containing extra environment variables for Airflow web pods\n ##\n extraEnvVarsCM: \"\"\n ## @param web.extraEnvVarsSecret Secret containing extra environment variables (in case of sensitive data) for Airflow web pods\n ##\n extraEnvVarsSecret: \"\"\n ## @param web.extraEnvVarsSecrets List of secrets with extra environment variables for Airflow web pods\n ##\n extraEnvVarsSecrets: []\n ## @param web.containerPorts.http Airflow web HTTP container port\n ##\n containerPorts:\n http: 8080\n ## @param web.replicaCount Number of Airflow web replicas\n ##\n replicaCount: 1\n ## Configure extra options for Airflow web containers' liveness, readiness and startup probes\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes\n ## @param web.livenessProbe.enabled Enable livenessProbe on Airflow web containers\n ## @param web.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe\n ## @param web.livenessProbe.periodSeconds Period seconds for livenessProbe\n ## @param web.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe\n ## @param web.livenessProbe.failureThreshold Failure threshold for livenessProbe\n ## @param web.livenessProbe.successThreshold Success threshold for livenessProbe\n ##\n livenessProbe:\n enabled: true\n initialDelaySeconds: 180\n periodSeconds: 20\n timeoutSeconds: 5\n failureThreshold: 6\n successThreshold: 1\n ## @param web.readinessProbe.enabled Enable readinessProbe on Airflow web containers\n ## @param web.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe\n ## @param web.readinessProbe.periodSeconds Period seconds for readinessProbe\n ## @param web.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe\n ## @param web.readinessProbe.failureThreshold Failure threshold for readinessProbe\n ## @param web.readinessProbe.successThreshold Success threshold for readinessProbe\n ##\n readinessProbe:\n enabled: true\n initialDelaySeconds: 30\n periodSeconds: 10\n timeoutSeconds: 5\n failureThreshold: 6\n successThreshold: 1\n ## @param web.startupProbe.enabled Enable startupProbe on Airflow web containers\n ## @param web.startupProbe.initialDelaySeconds Initial delay seconds for startupProbe\n ## @param web.startupProbe.periodSeconds Period seconds for startupProbe\n ## @param web.startupProbe.timeoutSeconds Timeout seconds for startupProbe\n ## @param web.startupProbe.failureThreshold Failure threshold for startupProbe\n ## @param web.startupProbe.successThreshold Success threshold for startupProbe\n ##\n startupProbe:\n enabled: false\n initialDelaySeconds: 60\n periodSeconds: 10\n timeoutSeconds: 1\n failureThreshold: 15\n successThreshold: 1\n ## @param web.customLivenessProbe Custom livenessProbe that overrides the default one\n ##\n customLivenessProbe: {}\n ## @param web.customReadinessProbe Custom readinessProbe that overrides the default one\n ##\n customReadinessProbe: {}\n ## @param web.customStartupProbe Custom startupProbe that overrides the default one\n ##\n customStartupProbe: {}\n ## Airflow web resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ## @param web.resources.limits The resources limits for the Airflow web containers\n ## @param web.resources.requests The requested resources for the Airflow web containers\n ##\n resources:\n limits: {}\n requests: {}\n ## Configure Airflow web pods Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod\n ## @param web.podSecurityContext.enabled Enabled Airflow web pods' Security Context\n ## @param web.podSecurityContext.fsGroup Set Airflow web pod's Security Context fsGroup\n ##\n podSecurityContext:\n enabled: true\n fsGroup: 1001\n ## Configure Airflow web containers (only main one) Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container\n ## @param web.containerSecurityContext.enabled Enabled Airflow web containers' Security Context\n ## @param web.containerSecurityContext.runAsUser Set Airflow web containers' Security Context runAsUser\n ## @param web.containerSecurityContext.runAsNonRoot Set Airflow web containers' Security Context runAsNonRoot\n ##\n containerSecurityContext:\n enabled: true\n runAsUser: 1001\n runAsNonRoot: true\n ## @param web.lifecycleHooks for the Airflow web container(s) to automate configuration before or after startup\n ##\n lifecycleHooks: {}\n ## @param web.hostAliases Deployment pod host aliases\n ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/\n ##\n hostAliases: []\n ## @param web.podLabels Add extra labels to the Airflow web pods\n ##\n podLabels: {}\n ## @param web.podAnnotations Add extra annotations to the Airflow web pods\n ##\n podAnnotations: {}\n ## @param web.affinity Affinity for Airflow web pods assignment (evaluated as a template)\n ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity\n ## Note: `web.podAffinityPreset`, `web.podAntiAffinityPreset`, and `web.nodeAffinityPreset` will be ignored when it's set\n ##\n affinity: {}\n ## Node affinity preset\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity\n ## @param web.nodeAffinityPreset.key Node label key to match. Ignored if `web.affinity` is set.\n ## @param web.nodeAffinityPreset.type Node affinity preset type. Ignored if `web.affinity` is set. Allowed values: `soft` or `hard`\n ## @param web.nodeAffinityPreset.values Node label values to match. Ignored if `web.affinity` is set.\n ##\n nodeAffinityPreset:\n ## e.g:\n ## key: \"kubernetes.io/e2e-az-name\"\n ##\n key: \"\"\n type: \"\"\n ## e.g:\n ## values:\n ## - e2e-az1\n ## - e2e-az2\n ##\n values: []\n ## @param web.nodeSelector Node labels for Airflow web pods assignment\n ## Ref: https://kubernetes.io/docs/user-guide/node-selection/\n ##\n nodeSelector: {}\n ## @param web.podAffinityPreset Pod affinity preset. Ignored if `web.affinity` is set. Allowed values: `soft` or `hard`.\n ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAffinityPreset: \"\"\n ## @param web.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `web.affinity` is set. Allowed values: `soft` or `hard`.\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAntiAffinityPreset: soft\n ## @param web.tolerations Tolerations for Airflow web pods assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n ##\n tolerations: []\n ## @param web.topologySpreadConstraints Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template\n ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods\n ##\n topologySpreadConstraints: []\n ## @param web.priorityClassName Priority Class Name\n ## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass\n ##\n priorityClassName: \"\"\n ## @param web.schedulerName Use an alternate scheduler, e.g. \"stork\".\n ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/\n ##\n schedulerName: \"\"\n ## @param web.terminationGracePeriodSeconds Seconds Airflow web pod needs to terminate gracefully\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods\n ##\n terminationGracePeriodSeconds: \"\"\n ## @param web.updateStrategy.type Airflow web deployment strategy type\n ## @param web.updateStrategy.rollingUpdate Airflow web deployment rolling update configuration parameters\n ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy\n ##\n updateStrategy:\n type: RollingUpdate\n rollingUpdate: {}\n ## @param web.sidecars Add additional sidecar containers to the Airflow web pods\n ## Example:\n ## sidecars:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n sidecars: []\n ## @param web.initContainers Add additional init containers to the Airflow web pods\n ## Example:\n ## initContainers:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n initContainers: []\n ## @param web.extraVolumeMounts Optionally specify extra list of additional volumeMounts for the Airflow web pods\n ##\n extraVolumeMounts: []\n ## @param web.extraVolumes Optionally specify extra list of additional volumes for the Airflow web pods\n ##\n extraVolumes: []\n ## Airflow web Pod Disruption Budget\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/\n ## @param web.pdb.create Deploy a pdb object for the Airflow web pods\n ## @param web.pdb.minAvailable Maximum number/percentage of unavailable Airflow web replicas\n ## @param web.pdb.maxUnavailable Maximum number/percentage of unavailable Airflow web replicas\n ##\n pdb:\n create: false\n minAvailable: 1\n maxUnavailable: \"\"\n\n## @section Airflow scheduler parameters\n\nscheduler:\n ## Bitnami Airflow Scheduler image version\n ## ref: https://hub.docker.com/r/bitnami/airflow-scheduler/tags/\n ## @param scheduler.image.registry Airflow Scheduler image registry\n ## @param scheduler.image.repository Airflow Scheduler image repository\n ## @param scheduler.image.tag Airflow Scheduler image tag (immutable tags are recommended)\n ## @param scheduler.image.digest Airflow Schefuler image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param scheduler.image.pullPolicy Airflow Scheduler image pull policy\n ## @param scheduler.image.pullSecrets Airflow Scheduler image pull secrets\n ## @param scheduler.image.debug Enable image debug mode\n ##\n image:\n registry: docker.io\n repository: bitnami/airflow-scheduler\n tag: 2.4.2-debian-11-r4\n digest: \"\"\n ## Specify a imagePullPolicy\n ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'\n ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images\n ##\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## Set to true if you would like to see extra information on logs\n ##\n debug: false\n ## @param scheduler.replicaCount Number of scheduler replicas\n ##\n replicaCount: 1\n ## @param scheduler.command Override cmd\n ##\n command: []\n ## @param scheduler.args Override args\n ##\n args: []\n ## @param scheduler.extraEnvVars Add extra environment variables\n ##\n extraEnvVars: []\n ## @param scheduler.extraEnvVarsCM ConfigMap with extra environment variables\n ##\n extraEnvVarsCM: \"\"\n ## @param scheduler.extraEnvVarsSecret Secret with extra environment variables\n ##\n extraEnvVarsSecret: \"\"\n ## @param scheduler.extraEnvVarsSecrets List of secrets with extra environment variables for Airflow scheduler pods\n ##\n extraEnvVarsSecrets: []\n ## @param scheduler.customLivenessProbe Custom livenessProbe that overrides the default one\n ##\n customLivenessProbe: {}\n ## @param scheduler.customReadinessProbe Custom readinessProbe that overrides the default one\n ##\n customReadinessProbe: {}\n ## @param scheduler.customStartupProbe Custom startupProbe that overrides the default one\n ##\n customStartupProbe: {}\n ## Airflow scheduler resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ## @param scheduler.resources.limits The resources limits for the Airflow scheduler containers\n ## @param scheduler.resources.requests The requested resources for the Airflow scheduler containers\n ##\n resources:\n limits: {}\n requests: {}\n ## Configure Airflow scheduler pods Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod\n ## @param scheduler.podSecurityContext.enabled Enabled Airflow scheduler pods' Security Context\n ## @param scheduler.podSecurityContext.fsGroup Set Airflow scheduler pod's Security Context fsGroup\n ##\n podSecurityContext:\n enabled: true\n fsGroup: 1001\n ## Configure Airflow scheduler containers (only main one) Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container\n ## @param scheduler.containerSecurityContext.enabled Enabled Airflow scheduler containers' Security Context\n ## @param scheduler.containerSecurityContext.runAsUser Set Airflow scheduler containers' Security Context runAsUser\n ## @param scheduler.containerSecurityContext.runAsNonRoot Set Airflow scheduler containers' Security Context runAsNonRoot\n ##\n containerSecurityContext:\n enabled: true\n runAsUser: 1001\n runAsNonRoot: true\n ## @param scheduler.lifecycleHooks for the Airflow scheduler container(s) to automate configuration before or after startup\n ##\n lifecycleHooks: {}\n ## @param scheduler.hostAliases Deployment pod host aliases\n ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/\n ##\n hostAliases: []\n ## @param scheduler.podLabels Add extra labels to the Airflow scheduler pods\n ##\n podLabels: {}\n ## @param scheduler.podAnnotations Add extra annotations to the Airflow scheduler pods\n ##\n podAnnotations: {}\n ## @param scheduler.affinity Affinity for Airflow scheduler pods assignment (evaluated as a template)\n ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity\n ## Note: `scheduler.podAffinityPreset`, `scheduler.podAntiAffinityPreset`, and `scheduler.nodeAffinityPreset` will be ignored when it's set\n ##\n affinity: {}\n ## Node affinity preset\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity\n ## @param scheduler.nodeAffinityPreset.key Node label key to match. Ignored if `scheduler.affinity` is set.\n ## @param scheduler.nodeAffinityPreset.type Node affinity preset type. Ignored if `scheduler.affinity` is set. Allowed values: `soft` or `hard`\n ## @param scheduler.nodeAffinityPreset.values Node label values to match. Ignored if `scheduler.affinity` is set.\n ##\n nodeAffinityPreset:\n ## e.g:\n ## key: \"kubernetes.io/e2e-az-name\"\n ##\n key: \"\"\n type: \"\"\n ## e.g:\n ## values:\n ## - e2e-az1\n ## - e2e-az2\n ##\n values: []\n ## @param scheduler.nodeSelector Node labels for Airflow scheduler pods assignment\n ## Ref: https://kubernetes.io/docs/user-guide/node-selection/\n ##\n nodeSelector: {}\n ## @param scheduler.podAffinityPreset Pod affinity preset. Ignored if `scheduler.affinity` is set. Allowed values: `soft` or `hard`.\n ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAffinityPreset: \"\"\n ## @param scheduler.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `scheduler.affinity` is set. Allowed values: `soft` or `hard`.\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAntiAffinityPreset: soft\n ## @param scheduler.tolerations Tolerations for Airflow scheduler pods assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n ##\n tolerations: []\n ## @param scheduler.topologySpreadConstraints Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template\n ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods\n ##\n topologySpreadConstraints: []\n ## @param scheduler.priorityClassName Priority Class Name\n ## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass\n ##\n priorityClassName: \"\"\n ## @param scheduler.schedulerName Use an alternate scheduler, e.g. \"stork\".\n ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/\n ##\n schedulerName: \"\"\n ## @param scheduler.terminationGracePeriodSeconds Seconds Airflow scheduler pod needs to terminate gracefully\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods\n ##\n terminationGracePeriodSeconds: \"\"\n ## @param scheduler.updateStrategy.type Airflow scheduler deployment strategy type\n ## @param scheduler.updateStrategy.rollingUpdate Airflow scheduler deployment rolling update configuration parameters\n ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy\n ##\n updateStrategy:\n type: RollingUpdate\n rollingUpdate: {}\n ## @param scheduler.sidecars Add additional sidecar containers to the Airflow scheduler pods\n ## Example:\n ## sidecars:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n sidecars: []\n ## @param scheduler.initContainers Add additional init containers to the Airflow scheduler pods\n ## Example:\n ## initContainers:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n initContainers: []\n ## @param scheduler.extraVolumeMounts Optionally specify extra list of additional volumeMounts for the Airflow scheduler pods\n ##\n extraVolumeMounts: []\n ## @param scheduler.extraVolumes Optionally specify extra list of additional volumes for the Airflow scheduler pods\n ##\n extraVolumes: []\n ## Airflow scheduler Pod Disruption Budget\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/\n ## @param scheduler.pdb.create Deploy a pdb object for the Airflow scheduler pods\n ## @param scheduler.pdb.minAvailable Maximum number/percentage of unavailable Airflow scheduler replicas\n ## @param scheduler.pdb.maxUnavailable Maximum number/percentage of unavailable Airflow scheduler replicas\n ##\n pdb:\n create: false\n minAvailable: 1\n maxUnavailable: \"\"\n\n## @section Airflow worker parameters\n\nworker:\n ## Bitnami Airflow Worker image version\n ## ref: https://hub.docker.com/r/bitnami/airflow-worker/tags/\n ## @param worker.image.registry Airflow Worker image registry\n ## @param worker.image.repository Airflow Worker image repository\n ## @param worker.image.tag Airflow Worker image tag (immutable tags are recommended)\n ## @param worker.image.digest Airflow Worker image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param worker.image.pullPolicy Airflow Worker image pull policy\n ## @param worker.image.pullSecrets Airflow Worker image pull secrets\n ## @param worker.image.debug Enable image debug mode\n ##\n image:\n registry: docker.io\n repository: bitnami/airflow-worker\n tag: 2.4.2-debian-11-r4\n digest: \"\"\n ## Specify a imagePullPolicy\n ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'\n ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images\n ##\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## Set to true if you would like to see extra information on logs\n ##\n debug: false\n ## @param worker.command Override default container command (useful when using custom images)\n ##\n command: []\n ## @param worker.args Override default container args (useful when using custom images)\n ##\n args: []\n ## @param worker.extraEnvVars Array with extra environment variables to add Airflow worker pods\n ##\n extraEnvVars: []\n ## @param worker.extraEnvVarsCM ConfigMap containing extra environment variables for Airflow worker pods\n ##\n extraEnvVarsCM: \"\"\n ## @param worker.extraEnvVarsSecret Secret containing extra environment variables (in case of sensitive data) for Airflow worker pods\n ##\n extraEnvVarsSecret: \"\"\n ## @param worker.extraEnvVarsSecrets List of secrets with extra environment variables for Airflow worker pods\n ##\n extraEnvVarsSecrets: []\n ## @param worker.containerPorts.http Airflow worker HTTP container port\n ##\n containerPorts:\n http: 8793\n ## @param worker.replicaCount Number of Airflow worker replicas\n ##\n replicaCount: 1\n ## Configure extra options for Airflow worker containers' liveness, readiness and startup probes\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes\n ## @param worker.livenessProbe.enabled Enable livenessProbe on Airflow worker containers\n ## @param worker.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe\n ## @param worker.livenessProbe.periodSeconds Period seconds for livenessProbe\n ## @param worker.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe\n ## @param worker.livenessProbe.failureThreshold Failure threshold for livenessProbe\n ## @param worker.livenessProbe.successThreshold Success threshold for livenessProbe\n ##\n livenessProbe:\n enabled: true\n initialDelaySeconds: 180\n periodSeconds: 20\n timeoutSeconds: 5\n failureThreshold: 6\n successThreshold: 1\n ## @param worker.readinessProbe.enabled Enable readinessProbe on Airflow worker containers\n ## @param worker.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe\n ## @param worker.readinessProbe.periodSeconds Period seconds for readinessProbe\n ## @param worker.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe\n ## @param worker.readinessProbe.failureThreshold Failure threshold for readinessProbe\n ## @param worker.readinessProbe.successThreshold Success threshold for readinessProbe\n ##\n readinessProbe:\n enabled: true\n initialDelaySeconds: 30\n periodSeconds: 10\n timeoutSeconds: 5\n failureThreshold: 6\n successThreshold: 1\n ## @param worker.startupProbe.enabled Enable startupProbe on Airflow worker containers\n ## @param worker.startupProbe.initialDelaySeconds Initial delay seconds for startupProbe\n ## @param worker.startupProbe.periodSeconds Period seconds for startupProbe\n ## @param worker.startupProbe.timeoutSeconds Timeout seconds for startupProbe\n ## @param worker.startupProbe.failureThreshold Failure threshold for startupProbe\n ## @param worker.startupProbe.successThreshold Success threshold for startupProbe\n ##\n startupProbe:\n enabled: false\n initialDelaySeconds: 60\n periodSeconds: 10\n timeoutSeconds: 1\n failureThreshold: 15\n successThreshold: 1\n ## @param worker.customLivenessProbe Custom livenessProbe that overrides the default one\n ##\n customLivenessProbe: {}\n ## @param worker.customReadinessProbe Custom readinessProbe that overrides the default one\n ##\n customReadinessProbe: {}\n ## @param worker.customStartupProbe Custom startupProbe that overrides the default one\n ##\n customStartupProbe: {}\n ## Airflow worker resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ## @param worker.resources.limits The resources limits for the Airflow worker containers\n ## @param worker.resources.requests The requested resources for the Airflow worker containers\n ##\n resources:\n limits: {}\n requests: {}\n ## Configure Airflow worker pods Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod\n ## @param worker.podSecurityContext.enabled Enabled Airflow worker pods' Security Context\n ## @param worker.podSecurityContext.fsGroup Set Airflow worker pod's Security Context fsGroup\n ##\n podSecurityContext:\n enabled: true\n fsGroup: 1001\n ## Configure Airflow worker containers (only main one) Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container\n ## @param worker.containerSecurityContext.enabled Enabled Airflow worker containers' Security Context\n ## @param worker.containerSecurityContext.runAsUser Set Airflow worker containers' Security Context runAsUser\n ## @param worker.containerSecurityContext.runAsNonRoot Set Airflow worker containers' Security Context runAsNonRoot\n ##\n containerSecurityContext:\n enabled: true\n runAsUser: 1001\n runAsNonRoot: true\n ## @param worker.lifecycleHooks for the Airflow worker container(s) to automate configuration before or after startup\n ##\n lifecycleHooks: {}\n ## @param worker.hostAliases Deployment pod host aliases\n ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/\n ##\n hostAliases: []\n ## @param worker.podLabels Add extra labels to the Airflow worker pods\n ##\n podLabels: {}\n ## @param worker.podAnnotations Add extra annotations to the Airflow worker pods\n ##\n podAnnotations: {}\n ## @param worker.affinity Affinity for Airflow worker pods assignment (evaluated as a template)\n ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity\n ## Note: `worker.podAffinityPreset`, `worker.podAntiAffinityPreset`, and `worker.nodeAffinityPreset` will be ignored when it's set\n ##\n affinity: {}\n ## Node affinity preset\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity\n ## @param worker.nodeAffinityPreset.key Node label key to match. Ignored if `worker.affinity` is set.\n ## @param worker.nodeAffinityPreset.type Node affinity preset type. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard`\n ## @param worker.nodeAffinityPreset.values Node label values to match. Ignored if `worker.affinity` is set.\n ##\n nodeAffinityPreset:\n ## e.g:\n ## key: \"kubernetes.io/e2e-az-name\"\n ##\n key: \"\"\n type: \"\"\n ## e.g:\n ## values:\n ## - e2e-az1\n ## - e2e-az2\n ##\n values: []\n ## @param worker.nodeSelector Node labels for Airflow worker pods assignment\n ## Ref: https://kubernetes.io/docs/user-guide/node-selection/\n ##\n nodeSelector: {}\n ## @param worker.podAffinityPreset Pod affinity preset. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard`.\n ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAffinityPreset: \"\"\n ## @param worker.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard`.\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAntiAffinityPreset: soft\n ## @param worker.tolerations Tolerations for Airflow worker pods assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n ##\n tolerations: []\n ## @param worker.topologySpreadConstraints Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template\n ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods\n ##\n topologySpreadConstraints: []\n ## @param worker.priorityClassName Priority Class Name\n ## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass\n ##\n priorityClassName: \"\"\n ## @param worker.schedulerName Use an alternate scheduler, e.g. \"stork\".\n ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/\n ##\n schedulerName: \"\"\n ## @param worker.terminationGracePeriodSeconds Seconds Airflow worker pod needs to terminate gracefully\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods\n ##\n terminationGracePeriodSeconds: \"\"\n ## @param worker.updateStrategy.type Airflow worker deployment strategy type\n ## @param worker.updateStrategy.rollingUpdate Airflow worker deployment rolling update configuration parameters\n ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy\n ##\n updateStrategy:\n type: RollingUpdate\n rollingUpdate: {}\n ## @param worker.sidecars Add additional sidecar containers to the Airflow worker pods\n ## Example:\n ## sidecars:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n sidecars: []\n ## @param worker.initContainers Add additional init containers to the Airflow worker pods\n ## Example:\n ## initContainers:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n initContainers: []\n ## @param worker.extraVolumeMounts Optionally specify extra list of additional volumeMounts for the Airflow worker pods\n ##\n extraVolumeMounts: []\n ## @param worker.extraVolumes Optionally specify extra list of additional volumes for the Airflow worker pods\n ##\n extraVolumes: []\n ## @param worker.extraVolumeClaimTemplates Optionally specify extra list of volumesClaimTemplates for the Airflow worker statefulset\n ##\n extraVolumeClaimTemplates: []\n ## @param worker.podTemplate Template to replace the default one to be use when `executor=KubernetesExecutor` to create Airflow worker pods\n ##\n podTemplate: {}\n ## Airflow worker Pod Disruption Budget\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/\n ## @param worker.pdb.create Deploy a pdb object for the Airflow worker pods\n ## @param worker.pdb.minAvailable Maximum number/percentage of unavailable Airflow worker replicas\n ## @param worker.pdb.maxUnavailable Maximum number/percentage of unavailable Airflow worker replicas\n ##\n pdb:\n create: false\n minAvailable: 1\n maxUnavailable: \"\"\n ## Enable HorizontalPodAutoscaler for worker pods\n ## ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/\n ## @param worker.autoscaling.enabled Whether enable horizontal pod autoscaler\n ## @param worker.autoscaling.minReplicas Configure a minimum amount of pods\n ## @param worker.autoscaling.maxReplicas Configure a maximum amount of pods\n ## @param worker.autoscaling.targetCPU Define the CPU target to trigger the scaling actions (utilization percentage)\n ## @param worker.autoscaling.targetMemory Define the memory target to trigger the scaling actions (utilization percentage)\n ##\n autoscaling:\n enabled: false\n minReplicas: 1\n maxReplicas: 3\n targetCPU: 80\n targetMemory: 80\n\n## @section Airflow git sync parameters\n\n## Configure Git to pull dags and plugins\n##\ngit:\n ## Bitnami Git image version\n ## ref: https://hub.docker.com/r/bitnami/git/tags/\n ## @param git.image.registry Git image registry\n ## @param git.image.repository Git image repository\n ## @param git.image.tag Git image tag (immutable tags are recommended)\n ## @param git.image.digest Git image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param git.image.pullPolicy Git image pull policy\n ## @param git.image.pullSecrets Git image pull secrets\n ##\n image:\n registry: docker.io\n repository: bitnami/git\n tag: 2.38.1-debian-11-r7\n digest: \"\"\n ## Specify a imagePullPolicy\n ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'\n ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images\n ##\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## Get DAG files from git repositories\n ## @param git.dags.enabled Enable in order to download DAG files from git repositories.\n ## @param git.dags.repositories [array] Array of repositories from which to download DAG files\n ##\n dags:\n enabled: false\n ## Name for repositories can be anything unique and must follow same naming conventions as kubernetes.\n ## Kubernetes resources can have names up to 253 characters long. The characters allowed in names are:\n ## digits (0-9), lower case letters (a-z), -, and .\n ## Example:\n ## - repository: https://github.com/myuser/myrepo\n ## branch: main\n ## name: my-dags\n ## path: /\n ##\n repositories:\n - repository: \"\"\n ## Branch from repository to checkout\n ##\n branch: \"\"\n ## An unique identifier for repository, must be unique for each repository\n ##\n name: \"\"\n ## Path to a folder in the repository containing the dags\n ##\n path: \"\"\n ## Get Plugins files from git repositories.\n ## @param git.plugins.enabled Enable in order to download Plugins files from git repositories.\n ## @param git.plugins.repositories [array] Array of repositories from which to download DAG files\n ##\n plugins:\n enabled: false\n repositories:\n - repository: \"\"\n ## Branch from repository to checkout\n ##\n branch: \"\"\n ## An unique identifier for repository, must be unique for each repository\n ##\n name: \"\"\n ## Path to a folder in the repository containing the plugins\n ##\n path: \"\"\n ## Properties for the Clone init container\n ## @param git.clone.command Override cmd\n ## @param git.clone.args Override args\n ## @param git.clone.extraVolumeMounts Add extra volume mounts\n ## @param git.clone.extraEnvVars Add extra environment variables\n ## @param git.clone.extraEnvVarsCM ConfigMap with extra environment variables\n ## @param git.clone.extraEnvVarsSecret Secret with extra environment variables\n ## @param git.clone.resources Clone init container resource requests and limits\n ##\n clone:\n command: []\n args: []\n extraVolumeMounts: []\n extraEnvVars: []\n extraEnvVarsCM: \"\"\n extraEnvVarsSecret: \"\"\n ## Clone init container resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ##\n resources: {}\n ## Properties for the Sync sidecar container\n ## @param git.sync.interval Interval in seconds to pull the git repository containing the plugins and/or DAG files\n ## @param git.sync.command Override cmd\n ## @param git.sync.args Override args\n ## @param git.sync.extraVolumeMounts Add extra volume mounts\n ## @param git.sync.extraEnvVars Add extra environment variables\n ## @param git.sync.extraEnvVarsCM ConfigMap with extra environment variables\n ## @param git.sync.extraEnvVarsSecret Secret with extra environment variables\n ## @param git.sync.resources Sync sidecar container resource requests and limits\n ##\n sync:\n interval: 60\n command: []\n args: []\n extraVolumeMounts: []\n extraEnvVars: []\n extraEnvVarsCM: \"\"\n extraEnvVarsSecret: \"\"\n ## Sync sidecar container resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ##\n resources: {}\n\n## @section Airflow ldap parameters\n\n## LDAP configuration\n## @param ldap.enabled Enable LDAP authentication\n## @param ldap.uri Server URI, eg. ldap://ldap_server:389\n## DEPRECATED ldap.base It will be removed in a future release, please use 'ldap.basedn' instead\n## @param ldap.basedn Base of the search, eg. ou=example,o=org.\n## DEPRECATED ldap.uidField It will be removed in a future release,, please use 'ldap.searchAttribute' instead\n## @param ldap.searchAttribute if doing an indirect bind to ldap, this is the field that matches the username when searching for the account to bind to\n## @param ldap.binddn DN of the account used to search in the LDAP server.\n## @param ldap.bindpw Bind Password\n## @param ldap.userRegistration Set to True to enable user self registration\n## @param ldap.userRegistrationRole Set role name to be assign when a user registers himself. This role must already exist. Mandatory when using ldap.userRegistration\n## @param ldap.rolesMapping mapping from LDAP DN to a list of roles\n## @param ldap.rolesSyncAtLogin replace ALL the user's roles each login, or only on registration\n##\nldap:\n enabled: false\n uri: \"ldap://ldap_server:389\"\n basedn: \"dc=example,dc=org\"\n searchAttribute: \"cn\"\n binddn: \"cn=admin,dc=example,dc=org\"\n bindpw: \"\"\n userRegistration: 'True'\n userRegistrationRole: \"Public\"\n rolesMapping: '{ \"cn=All,ou=Groups,dc=example,dc=org\": [\"User\"], \"cn=Admins,ou=Groups,dc=example,dc=org\": [\"Admin\"], }'\n rolesSyncAtLogin: 'True'\n\n ## SSL/TLS parameters for LDAP\n ## @param ldap.tls.enabled Enabled TLS/SSL for LDAP, you must include the CA file.\n ## @param ldap.tls.allowSelfSigned Allow to use self signed certificates\n ## DEPRECATED ldap.tls.CAcertificateSecret It will be removed in a future release, please use ldap.tls.certificatesSecret instead\n ## @param ldap.tls.certificatesSecret Name of the existing secret containing the certificate CA file that will be used by ldap client\n ## @param ldap.tls.certificatesMountPath Where LDAP certifcates are mounted.\n ## DEPRECATED ldap.tls.CAcertificateFilename It will be removed in a future release, please use ldap.tls.CAFilename instead\n ## @param ldap.tls.CAFilename LDAP CA cert filename\n ##\n tls:\n enabled: false\n allowSelfSigned: true\n certificatesSecret: \"\"\n certificatesMountPath: /opt/bitnami/airflow/conf/certs\n CAFilename: \"\"\n\n## @section Traffic Exposure Parameters\n\n## Airflow service parameters\n##\nservice:\n ## @param service.type Airflow service type\n ##\n type: ClusterIP\n ## @param service.ports.http Airflow service HTTP port\n ##\n ports:\n http: 8080\n ## Node ports to expose\n ## @param service.nodePorts.http Node port for HTTP\n ## NOTE: choose port between <30000-32767>\n ##\n nodePorts:\n http: \"\"\n ## @param service.sessionAffinity Control where client requests go, to the same pod or round-robin\n ## Values: ClientIP or None\n ## ref: https://kubernetes.io/docs/user-guide/services/\n ##\n sessionAffinity: None\n ## @param service.sessionAffinityConfig Additional settings for the sessionAffinity\n ## sessionAffinityConfig:\n ## clientIP:\n ## timeoutSeconds: 300\n ##\n sessionAffinityConfig: {}\n ## @param service.clusterIP Airflow service Cluster IP\n ## e.g.:\n ## clusterIP: None\n ##\n clusterIP: \"\"\n ## @param service.loadBalancerIP Airflow service Load Balancer IP\n ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-loadbalancer\n ##\n loadBalancerIP: \"\"\n ## @param service.loadBalancerSourceRanges Airflow service Load Balancer sources\n ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service\n ## e.g:\n ## loadBalancerSourceRanges:\n ## - 10.10.10.0/24\n ##\n loadBalancerSourceRanges: []\n ## @param service.externalTrafficPolicy Airflow service external traffic policy\n ## ref https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip\n ##\n externalTrafficPolicy: Cluster\n ## @param service.annotations Additional custom annotations for Airflow service\n ##\n annotations: {}\n ## @param service.extraPorts Extra port to expose on Airflow service\n ##\n extraPorts: []\n\n## Airflow ingress parameters\n## ref: https://kubernetes.io/docs/user-guide/ingress/\n##\ningress:\n ## @param ingress.enabled Enable ingress record generation for Airflow\n ##\n enabled: false\n ## @param ingress.ingressClassName IngressClass that will be be used to implement the Ingress (Kubernetes 1.18+)\n ## This is supported in Kubernetes 1.18+ and required if you have more than one IngressClass marked as the default for your cluster .\n ## ref: https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/\n ##\n ingressClassName: \"\"\n ## @param ingress.pathType Ingress path type\n ##\n pathType: ImplementationSpecific\n ## @param ingress.apiVersion Force Ingress API version (automatically detected if not set)\n ##\n apiVersion: \"\"\n ## @param ingress.hostname Default host for the ingress record\n ##\n hostname: airflow.local\n ## @param ingress.path Default path for the ingress record\n ## NOTE: You may need to set this to '/*' in order to use this with ALB ingress controllers\n ##\n path: /\n ## @param ingress.annotations [object] Additional annotations for the Ingress resource. To enable certificate autogeneration, place here your cert-manager annotations.\n ## Use this parameter to set the required annotations for cert-manager, see\n ## ref: https://cert-manager.io/docs/usage/ingress/#supported-annotations\n ## e.g:\n ## annotations:\n ## kubernetes.io/ingress.class: nginx\n ## cert-manager.io/cluster-issuer: cluster-issuer-name\n ##\n annotations: {}\n ## @param ingress.tls Enable TLS configuration for the host defined at `ingress.hostname` parameter\n ## TLS certificates will be retrieved from a TLS secret with name: `{{- printf \"%s-tls\" .Values.ingress.hostname }}`\n ## You can:\n ## - Use the `ingress.secrets` parameter to create this TLS secret\n ## - Rely on cert-manager to create it by setting the corresponding annotations\n ## - Rely on Helm to create self-signed certificates by setting `ingress.selfSigned=true`\n ##\n tls: false\n ## @param ingress.selfSigned Create a TLS secret for this ingress record using self-signed certificates generated by Helm\n ##\n selfSigned: false\n ## @param ingress.extraHosts An array with additional hostname(s) to be covered with the ingress record\n ## e.g:\n ## extraHosts:\n ## - name: airflow.local\n ## path: /\n ##\n extraHosts: []\n ## @param ingress.extraPaths An array with additional arbitrary paths that may need to be added to the ingress under the main host\n ## e.g:\n ## extraPaths:\n ## - path: /*\n ## backend:\n ## serviceName: ssl-redirect\n ## servicePort: use-annotation\n ##\n extraPaths: []\n ## @param ingress.extraTls TLS configuration for additional hostname(s) to be covered with this ingress record\n ## ref: https://kubernetes.io/docs/concepts/services-networking/ingress/#tls\n ## e.g:\n ## extraTls:\n ## - hosts:\n ## - airflow.local\n ## secretName: airflow.local-tls\n ##\n extraTls: []\n ## @param ingress.secrets Custom TLS certificates as secrets\n ## NOTE: 'key' and 'certificate' are expected in PEM format\n ## NOTE: 'name' should line up with a 'secretName' set further up\n ## If it is not set and you're using cert-manager, this is unneeded, as it will create a secret for you with valid certificates\n ## If it is not set and you're NOT using cert-manager either, self-signed certificates will be created valid for 365 days\n ## It is also possible to create and manage the certificates outside of this helm chart\n ## Please see README.md for more information\n ## e.g:\n ## secrets:\n ## - name: airflow.local-tls\n ## key: |-\n ## -----BEGIN RSA PRIVATE KEY-----\n ## ...\n ## -----END RSA PRIVATE KEY-----\n ## certificate: |-\n ## -----BEGIN CERTIFICATE-----\n ## ...\n ## -----END CERTIFICATE-----\n ##\n secrets: []\n ## @param ingress.extraRules Additional rules to be covered with this ingress record\n ## ref: https://kubernetes.io/docs/concepts/services-networking/ingress/#ingress-rules\n ## e.g:\n ## extraRules:\n ## - host: example.local\n ## http:\n ## path: /\n ## backend:\n ## service:\n ## name: example-svc\n ## port:\n ## name: http\n ##\n extraRules: []\n\n## @section Other Parameters\n\n## Service account for Airflow pods to use.\n## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/\n##\nserviceAccount:\n ## @param serviceAccount.create Enable creation of ServiceAccount for Airflow pods\n ##\n create: false\n ## @param serviceAccount.name The name of the ServiceAccount to use.\n ## If not set and create is true, a name is generated using the common.names.fullname template\n ##\n name: \"\"\n ## @param serviceAccount.automountServiceAccountToken Allows auto mount of ServiceAccountToken on the serviceAccount created\n ## Can be set to false if pods using this serviceAccount do not need to use K8s API\n ##\n automountServiceAccountToken: true\n ## @param serviceAccount.annotations Additional custom annotations for the ServiceAccount\n ##\n annotations: {}\n## Role Based Access\n## Ref: https://kubernetes.io/docs/admin/authorization/rbac/\n## @param rbac.create Create Role and RoleBinding\n##\nrbac:\n create: false\n ## @param rbac.rules Custom RBAC rules to set\n ## e.g:\n ## rules:\n ## - apiGroups:\n ## - \"\"\n ## resources:\n ## - pods\n ## verbs:\n ## - get\n ## - list\n ##\n rules: []\n\n## @section Airflow metrics parameters\n\nmetrics:\n ## @param metrics.enabled Whether or not to create a standalone Airflow exporter to expose Airflow metrics\n ##\n enabled: false\n ## Bitnami Airflow exporter image\n ## ref: https://hub.docker.com/r/bitnami/airflow-exporter/tags/\n ## @param metrics.image.registry Airflow exporter image registry\n ## @param metrics.image.repository Airflow exporter image repository\n ## @param metrics.image.tag Airflow exporter image tag (immutable tags are recommended)\n ## @param metrics.image.digest Airflow exporter image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param metrics.image.pullPolicy Airflow exporter image pull policy\n ## @param metrics.image.pullSecrets Airflow exporter image pull secrets\n ##\n image:\n registry: docker.io\n repository: bitnami/airflow-exporter\n tag: 0.20220314.0-debian-11-r57\n digest: \"\"\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## @param metrics.extraEnvVars Array with extra environment variables to add Airflow exporter pods\n ##\n extraEnvVars: []\n ## @param metrics.extraEnvVarsCM ConfigMap containing extra environment variables for Airflow exporter pods\n ##\n extraEnvVarsCM: \"\"\n ## @param metrics.extraEnvVarsSecret Secret containing extra environment variables (in case of sensitive data) for Airflow exporter pods\n ##\n extraEnvVarsSecret: \"\"\n ## @param metrics.containerPorts.http Airflow exporter metrics container port\n ##\n containerPorts:\n http: 9112\n ## Airflow exporter resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ## @param metrics.resources.limits The resources limits for the container\n ## @param metrics.resources.requests The requested resources for the container\n ##\n resources:\n limits: {}\n requests: {}\n ## Airflow exporter pods' Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod\n ## @param metrics.podSecurityContext.enabled Enable security context for the pods\n ## @param metrics.podSecurityContext.fsGroup Set Airflow exporter pod's Security Context fsGroup\n ##\n podSecurityContext:\n enabled: true\n fsGroup: 1001\n ## Airflow exporter containers' Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container\n ## @param metrics.containerSecurityContext.enabled Enable Airflow exporter containers' Security Context\n ## @param metrics.containerSecurityContext.runAsUser Set Airflow exporter containers' Security Context runAsUser\n ## @param metrics.containerSecurityContext.runAsNonRoot Set Airflow exporter containers' Security Context runAsNonRoot\n ## e.g:\n ## containerSecurityContext:\n ## enabled: true\n ## capabilities:\n ## drop: [\"NET_RAW\"]\n ## readOnlyRootFilesystem: true\n ##\n containerSecurityContext:\n enabled: true\n runAsUser: 1001\n runAsNonRoot: true\n ## @param metrics.lifecycleHooks for the Airflow exporter container(s) to automate configuration before or after startup\n ##\n lifecycleHooks: {}\n ## @param metrics.hostAliases Airflow exporter pods host aliases\n ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/\n ##\n hostAliases: []\n ## @param metrics.podLabels Extra labels for Airflow exporter pods\n ## Ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/\n ##\n podLabels: {}\n ## @param metrics.podAnnotations Extra annotations for Airflow exporter pods\n ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/\n ##\n podAnnotations: {}\n ## @param metrics.podAffinityPreset Pod affinity preset. Ignored if `metrics.affinity` is set. Allowed values: `soft` or `hard`\n ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAffinityPreset: \"\"\n ## @param metrics.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `metrics.affinity` is set. Allowed values: `soft` or `hard`\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAntiAffinityPreset: soft\n ## Node metrics.affinity preset\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity\n ##\n nodeAffinityPreset:\n ## @param metrics.nodeAffinityPreset.type Node affinity preset type. Ignored if `metrics.affinity` is set. Allowed values: `soft` or `hard`\n ##\n type: \"\"\n ## @param metrics.nodeAffinityPreset.key Node label key to match Ignored if `metrics.affinity` is set.\n ## E.g.\n ## key: \"kubernetes.io/e2e-az-name\"\n ##\n key: \"\"\n ## @param metrics.nodeAffinityPreset.values Node label values to match. Ignored if `metrics.affinity` is set.\n ## E.g.\n ## values:\n ## - e2e-az1\n ## - e2e-az2\n ##\n values: []\n ## @param metrics.affinity Affinity for pod assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity\n ## Note: metrics.podAffinityPreset, metrics.podAntiAffinityPreset, and metrics.nodeAffinityPreset will be ignored when it's set\n ##\n affinity: {}\n ## @param metrics.nodeSelector Node labels for pod assignment\n ## Ref: https://kubernetes.io/docs/user-guide/node-selection/\n ##\n nodeSelector: {}\n ## @param metrics.tolerations Tolerations for pod assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n ##\n tolerations: []\n ## @param metrics.schedulerName Name of the k8s scheduler (other than default) for Airflow exporter\n ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/\n ##\n schedulerName: \"\"\n ## Airflow exporter service configuration\n ##\n service:\n ## @param metrics.service.ports.http Airflow exporter metrics service port\n ##\n ports:\n http: 9112\n ## @param metrics.service.clusterIP Static clusterIP or None for headless services\n ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#choosing-your-own-ip-address\n ##\n clusterIP: \"\"\n ## @param metrics.service.sessionAffinity Control where client requests go, to the same pod or round-robin\n ## Values: ClientIP or None\n ## ref: https://kubernetes.io/docs/user-guide/services/\n ##\n sessionAffinity: None\n ## @param metrics.service.annotations [object] Annotations for the Airflow exporter service\n ##\n annotations:\n prometheus.io/scrape: \"true\"\n prometheus.io/port: \"{{ .Values.metrics.service.ports.http }}\"\n ## Prometheus Operator ServiceMonitor configuration\n ##\n serviceMonitor:\n ## @param metrics.serviceMonitor.enabled if `true`, creates a Prometheus Operator ServiceMonitor (requires `metrics.enabled` to be `true`)\n ##\n enabled: false\n ## @param metrics.serviceMonitor.namespace Namespace in which Prometheus is running\n ##\n namespace: \"\"\n ## @param metrics.serviceMonitor.interval Interval at which metrics should be scraped\n ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint\n ##\n interval: \"\"\n ## @param metrics.serviceMonitor.scrapeTimeout Timeout after which the scrape is ended\n ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint\n ##\n scrapeTimeout: \"\"\n ## @param metrics.serviceMonitor.labels Additional labels that can be used so ServiceMonitor will be discovered by Prometheus\n ##\n labels: {}\n ## @param metrics.serviceMonitor.selector Prometheus instance selector labels\n ## ref: https://github.com/bitnami/charts/tree/main/bitnami/prometheus-operator#prometheus-configuration\n ##\n selector: {}\n ## @param metrics.serviceMonitor.relabelings RelabelConfigs to apply to samples before scraping\n ##\n relabelings: []\n ## @param metrics.serviceMonitor.metricRelabelings MetricRelabelConfigs to apply to samples before ingestion\n ##\n metricRelabelings: []\n ## @param metrics.serviceMonitor.honorLabels Specify honorLabels parameter to add the scrape endpoint\n ##\n honorLabels: false\n ## @param metrics.serviceMonitor.jobLabel The name of the label on the target service to use as the job name in prometheus.\n ##\n jobLabel: \"\"\n\n## @section Airflow database parameters\n\n## PostgreSQL chart configuration\n## ref: https://github.com/bitnami/charts/blob/main/bitnami/postgresql/values.yaml\n## @param postgresql.enabled Switch to enable or disable the PostgreSQL helm chart\n## @param postgresql.auth.enablePostgresUser Assign a password to the \"postgres\" admin user. Otherwise, remote access will be blocked for this user\n## @param postgresql.auth.username Name for a custom user to create\n## @param postgresql.auth.password Password for the custom user to create\n## @param postgresql.auth.database Name for a custom database to create\n## @param postgresql.auth.existingSecret Name of existing secret to use for PostgreSQL credentials\n## @param postgresql.architecture PostgreSQL architecture (`standalone` or `replication`)\n##\npostgresql:\n enabled: true\n auth:\n enablePostgresUser: false\n username: bn_airflow\n password: \"\"\n database: bitnami_airflow\n existingSecret: \"\"\n architecture: standalone\n## External PostgreSQL configuration\n## All of these values are only used when postgresql.enabled is set to false\n## @param externalDatabase.host Database host\n## @param externalDatabase.port Database port number\n## @param externalDatabase.user Non-root username for Airflow\n## @param externalDatabase.password Password for the non-root username for Airflow\n## @param externalDatabase.database Airflow database name\n## @param externalDatabase.existingSecret Name of an existing secret resource containing the database credentials\n## @param externalDatabase.existingSecretPasswordKey Name of an existing secret key containing the database credentials\n##\nexternalDatabase:\n host: localhost\n port: 5432\n user: bn_airflow\n database: bitnami_airflow\n password: \"\"\n existingSecret: \"\"\n existingSecretPasswordKey: \"\"\n\n## Redis® chart configuration\n## ref: https://github.com/bitnami/charts/blob/main/bitnami/redis/values.yaml\n## @param redis.enabled Switch to enable or disable the Redis® helm\n## @param redis.auth.enabled Enable password authentication\n## @param redis.auth.password Redis® password\n## @param redis.auth.existingSecret The name of an existing secret with Redis® credentials\n## @param redis.architecture Redis® architecture. Allowed values: `standalone` or `replication`\n##\nredis:\n enabled: true\n auth:\n enabled: true\n ## Redis® password (both master and slave). Defaults to a random 10-character alphanumeric string if not set and auth.enabled is true.\n ## It should always be set using the password value or in the existingSecret to avoid issues\n ## with Airflow.\n ## The password value is ignored if existingSecret is set\n password: \"\"\n existingSecret: \"\"\n architecture: standalone\n\n## External Redis® configuration\n## All of these values are only used when redis.enabled is set to false\n## @param externalRedis.host Redis® host\n## @param externalRedis.port Redis® port number\n## @param externalRedis.username Redis® username\n## @param externalRedis.password Redis® password\n## @param externalRedis.existingSecret Name of an existing secret resource containing the Redis&trade credentials\n## @param externalRedis.existingSecretPasswordKey Name of an existing secret key containing the Redis&trade credentials\n##\nexternalRedis:\n host: localhost\n port: 6379\n ## Most Redis® implementations do not require a username\n ## to authenticate and it should be enough with the password\n username: \"\"\n password: \"\"\n existingSecret: \"\"\n existingSecretPasswordKey: \"\"\n", + ValuesOverrideYaml: "## @section Global parameters\n## Global Docker image parameters\n## Please, note that this will override the image parameters, including dependencies, configured to use the global value\n## Current available global Docker image parameters: imageRegistry, imagePullSecrets and storageClass\n\n## @param global.imageRegistry Global Docker image registry\n## @param global.imagePullSecrets Global Docker registry secret names as an array\n## @param global.storageClass Global StorageClass for Persistent Volume(s)\n##\nglobal:\n imageRegistry: \"\"\n ## E.g.\n ## imagePullSecrets:\n ## - myRegistryKeySecretName\n ##\n imagePullSecrets: []\n storageClass: \"\"\n\n## @section Common parameters\n\n## @param kubeVersion Override Kubernetes version\n##\nkubeVersion: \"\"\n## @param nameOverride String to partially override common.names.fullname template (will maintain the release name)\n##\nnameOverride: \"\"\n## @param fullnameOverride String to fully override common.names.fullname template\n##\nfullnameOverride: \"\"\n## @param clusterDomain Kubernetes Cluster Domain\n##\nclusterDomain: cluster.local\n## @param extraDeploy Extra objects to deploy (evaluated as a template)\n##\nextraDeploy: []\n## @param commonLabels Add labels to all the deployed resources\n##\ncommonLabels: {}\n## @param commonAnnotations Add annotations to all the deployed resources\n##\ncommonAnnotations: {}\n## Enable diagnostic mode in the deployment(s)/statefulset(s)\n##\ndiagnosticMode:\n ## @param diagnosticMode.enabled Enable diagnostic mode (all probes will be disabled and the command will be overridden)\n ##\n enabled: false\n ## @param diagnosticMode.command Command to override all containers in the the deployment(s)/statefulset(s)\n ##\n command:\n - sleep\n ## @param diagnosticMode.args Args to override all containers in the the deployment(s)/statefulset(s)\n ##\n args:\n - infinity\n\n## @section Airflow common parameters\n\n## Authentication parameters\n## ref: https://github.com/bitnami/containers/tree/main/bitnami/airflow#environment-variables\n##\nauth:\n ## @param auth.username Username to access web UI\n ##\n username: user\n ## @param auth.password Password to access web UI\n ##\n password: \"\"\n ## @param auth.fernetKey Fernet key to secure connections\n ## ref: https://airflow.readthedocs.io/en/stable/howto/secure-connections.html\n ## ref: https://bcb.github.io/airflow/fernet-key\n ##\n fernetKey: \"\"\n ## @param auth.secretKey Secret key to run your flask app\n ## ref: https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.html#secret-key\n ##\n secretKey: \"\"\n ## @param auth.existingSecret Name of an existing secret to use for Airflow credentials\n ## `auth.password`, `auth.fernetKey`, and `auth.secretKey` will be ignored and picked up from this secret\n ## The secret must contain the keys `airflow-password`, `airflow-fernet-key` and `airflow-secret-key'\n ## The value is evaluated as a template\n ##\n existingSecret: \"\"\n## @param executor Airflow executor. Allowed values: `SequentialExecutor`, `LocalExecutor`, `CeleryExecutor`, `KubernetesExecutor`, `CeleryKubernetesExecutor` and `LocalKubernetesExecutor`\n## ref: http://airflow.apache.org/docs/stable/executor/index.html\n##\nexecutor: CeleryExecutor\n## @param loadExamples Switch to load some Airflow examples\n##\nloadExamples: false\n## @param configuration Specify content for Airflow config file (auto-generated based on other env. vars otherwise)\n## e.g:\n## configuration: |-\n## [core]\n## dags_folder=/opt/bitnami/airflow/dags\n## ...\n##\nconfiguration: \"\"\n## @param existingConfigmap Name of an existing ConfigMap with the Airflow config file\n##\nexistingConfigmap: \"\"\n## Load custom DAGs from a ConfigMap\n## Note: an init container will be used to prepare the DAGs available in the ConfigMap to be consumed by Airflow containers\n##\ndags:\n ## @param dags.existingConfigmap Name of an existing ConfigMap with all the DAGs files you want to load in Airflow\n ##\n existingConfigmap: \"\"\n ## Bitnami Shell image\n ## ref: https://hub.docker.com/r/bitnami/bitnami-shell/tags/\n ## @param dags.image.registry Init container load-dags image registry\n ## @param dags.image.repository Init container load-dags image repository\n ## @param dags.image.tag Init container load-dags image tag (immutable tags are recommended)\n ## @param dags.image.digest Init container load-dags image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param dags.image.pullPolicy Init container load-dags image pull policy\n ## @param dags.image.pullSecrets Init container load-dags image pull secrets\n ##\n image:\n registry: docker.io\n repository: bitnami/bitnami-shell\n tag: 11-debian-11-r50\n digest: \"\"\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n## @param extraEnvVars Add extra environment variables for all the Airflow pods\n##\nextraEnvVars: []\n## @param extraEnvVarsCM ConfigMap with extra environment variables for all the Airflow pods\n##\nextraEnvVarsCM: \"\"\n## @param extraEnvVarsSecret Secret with extra environment variables for all the Airflow pods\n##\nextraEnvVarsSecret: \"\"\n## @param extraEnvVarsSecrets List of secrets with extra environment variables for all the Airflow pods\n##\nextraEnvVarsSecrets: []\n## @param sidecars Add additional sidecar containers to all the Airflow pods\n## Example:\n## sidecars:\n## - name: your-image-name\n## image: your-image\n## imagePullPolicy: Always\n## ports:\n## - name: portname\n## containerPort: 1234\n##\nsidecars: []\n## @param initContainers Add additional init containers to all the Airflow pods\n## Example:\n## initContainers:\n## - name: your-image-name\n## image: your-image\n## imagePullPolicy: Always\n## ports:\n## - name: portname\n## containerPort: 1234\n##\ninitContainers: []\n## @param extraVolumeMounts Optionally specify extra list of additional volumeMounts for all the Airflow pods\n##\nextraVolumeMounts: []\n## @param extraVolumes Optionally specify extra list of additional volumes for the all the Airflow pods\n##\nextraVolumes: []\n\n## @section Airflow web parameters\n\nweb:\n ## Bitnami Airflow image version\n ## ref: https://hub.docker.com/r/bitnami/airflow/tags/\n ## @param web.image.registry Airflow image registry\n ## @param web.image.repository Airflow image repository\n ## @param web.image.tag Airflow image tag (immutable tags are recommended)\n ## @param web.image.digest Airflow image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param web.image.pullPolicy Airflow image pull policy\n ## @param web.image.pullSecrets Airflow image pull secrets\n ## @param web.image.debug Enable image debug mode\n image:\n registry: docker.io\n repository: bitnami/airflow\n tag: 2.4.2-debian-11-r6\n digest: \"\"\n ## Specify a imagePullPolicy\n ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'\n ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images\n ##\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## Set to true if you would like to see extra information on logs\n ##\n debug: false\n ## @param web.baseUrl URL used to access to Airflow web ui\n ##\n baseUrl: \"\"\n ## @param web.existingConfigmap Name of an existing config map containing the Airflow web config file\n ##\n existingConfigmap: \"\"\n ## @param web.command Override default container command (useful when using custom images)\n ##\n command: []\n ## @param web.args Override default container args (useful when using custom images)\n ##\n args: []\n ## @param web.extraEnvVars Array with extra environment variables to add Airflow web pods\n ##\n extraEnvVars: []\n ## @param web.extraEnvVarsCM ConfigMap containing extra environment variables for Airflow web pods\n ##\n extraEnvVarsCM: \"\"\n ## @param web.extraEnvVarsSecret Secret containing extra environment variables (in case of sensitive data) for Airflow web pods\n ##\n extraEnvVarsSecret: \"\"\n ## @param web.extraEnvVarsSecrets List of secrets with extra environment variables for Airflow web pods\n ##\n extraEnvVarsSecrets: []\n ## @param web.containerPorts.http Airflow web HTTP container port\n ##\n containerPorts:\n http: 8080\n ## @param web.replicaCount Number of Airflow web replicas\n ##\n replicaCount: 1\n ## Configure extra options for Airflow web containers' liveness, readiness and startup probes\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes\n ## @param web.livenessProbe.enabled Enable livenessProbe on Airflow web containers\n ## @param web.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe\n ## @param web.livenessProbe.periodSeconds Period seconds for livenessProbe\n ## @param web.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe\n ## @param web.livenessProbe.failureThreshold Failure threshold for livenessProbe\n ## @param web.livenessProbe.successThreshold Success threshold for livenessProbe\n ##\n livenessProbe:\n enabled: true\n initialDelaySeconds: 180\n periodSeconds: 20\n timeoutSeconds: 5\n failureThreshold: 6\n successThreshold: 1\n ## @param web.readinessProbe.enabled Enable readinessProbe on Airflow web containers\n ## @param web.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe\n ## @param web.readinessProbe.periodSeconds Period seconds for readinessProbe\n ## @param web.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe\n ## @param web.readinessProbe.failureThreshold Failure threshold for readinessProbe\n ## @param web.readinessProbe.successThreshold Success threshold for readinessProbe\n ##\n readinessProbe:\n enabled: true\n initialDelaySeconds: 30\n periodSeconds: 10\n timeoutSeconds: 5\n failureThreshold: 6\n successThreshold: 1\n ## @param web.startupProbe.enabled Enable startupProbe on Airflow web containers\n ## @param web.startupProbe.initialDelaySeconds Initial delay seconds for startupProbe\n ## @param web.startupProbe.periodSeconds Period seconds for startupProbe\n ## @param web.startupProbe.timeoutSeconds Timeout seconds for startupProbe\n ## @param web.startupProbe.failureThreshold Failure threshold for startupProbe\n ## @param web.startupProbe.successThreshold Success threshold for startupProbe\n ##\n startupProbe:\n enabled: false\n initialDelaySeconds: 60\n periodSeconds: 10\n timeoutSeconds: 1\n failureThreshold: 15\n successThreshold: 1\n ## @param web.customLivenessProbe Custom livenessProbe that overrides the default one\n ##\n customLivenessProbe: {}\n ## @param web.customReadinessProbe Custom readinessProbe that overrides the default one\n ##\n customReadinessProbe: {}\n ## @param web.customStartupProbe Custom startupProbe that overrides the default one\n ##\n customStartupProbe: {}\n ## Airflow web resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ## @param web.resources.limits The resources limits for the Airflow web containers\n ## @param web.resources.requests The requested resources for the Airflow web containers\n ##\n resources:\n limits: {}\n requests: {}\n ## Configure Airflow web pods Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod\n ## @param web.podSecurityContext.enabled Enabled Airflow web pods' Security Context\n ## @param web.podSecurityContext.fsGroup Set Airflow web pod's Security Context fsGroup\n ##\n podSecurityContext:\n enabled: true\n fsGroup: 1001\n ## Configure Airflow web containers (only main one) Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container\n ## @param web.containerSecurityContext.enabled Enabled Airflow web containers' Security Context\n ## @param web.containerSecurityContext.runAsUser Set Airflow web containers' Security Context runAsUser\n ## @param web.containerSecurityContext.runAsNonRoot Set Airflow web containers' Security Context runAsNonRoot\n ##\n containerSecurityContext:\n enabled: true\n runAsUser: 1001\n runAsNonRoot: true\n ## @param web.lifecycleHooks for the Airflow web container(s) to automate configuration before or after startup\n ##\n lifecycleHooks: {}\n ## @param web.hostAliases Deployment pod host aliases\n ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/\n ##\n hostAliases: []\n ## @param web.podLabels Add extra labels to the Airflow web pods\n ##\n podLabels: {}\n ## @param web.podAnnotations Add extra annotations to the Airflow web pods\n ##\n podAnnotations: {}\n ## @param web.affinity Affinity for Airflow web pods assignment (evaluated as a template)\n ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity\n ## Note: `web.podAffinityPreset`, `web.podAntiAffinityPreset`, and `web.nodeAffinityPreset` will be ignored when it's set\n ##\n affinity: {}\n ## Node affinity preset\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity\n ## @param web.nodeAffinityPreset.key Node label key to match. Ignored if `web.affinity` is set.\n ## @param web.nodeAffinityPreset.type Node affinity preset type. Ignored if `web.affinity` is set. Allowed values: `soft` or `hard`\n ## @param web.nodeAffinityPreset.values Node label values to match. Ignored if `web.affinity` is set.\n ##\n nodeAffinityPreset:\n ## e.g:\n ## key: \"kubernetes.io/e2e-az-name\"\n ##\n key: \"\"\n type: \"\"\n ## e.g:\n ## values:\n ## - e2e-az1\n ## - e2e-az2\n ##\n values: []\n ## @param web.nodeSelector Node labels for Airflow web pods assignment\n ## Ref: https://kubernetes.io/docs/user-guide/node-selection/\n ##\n nodeSelector: {}\n ## @param web.podAffinityPreset Pod affinity preset. Ignored if `web.affinity` is set. Allowed values: `soft` or `hard`.\n ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAffinityPreset: \"\"\n ## @param web.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `web.affinity` is set. Allowed values: `soft` or `hard`.\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAntiAffinityPreset: soft\n ## @param web.tolerations Tolerations for Airflow web pods assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n ##\n tolerations: []\n ## @param web.topologySpreadConstraints Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template\n ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods\n ##\n topologySpreadConstraints: []\n ## @param web.priorityClassName Priority Class Name\n ## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass\n ##\n priorityClassName: \"\"\n ## @param web.schedulerName Use an alternate scheduler, e.g. \"stork\".\n ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/\n ##\n schedulerName: \"\"\n ## @param web.terminationGracePeriodSeconds Seconds Airflow web pod needs to terminate gracefully\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods\n ##\n terminationGracePeriodSeconds: \"\"\n ## @param web.updateStrategy.type Airflow web deployment strategy type\n ## @param web.updateStrategy.rollingUpdate Airflow web deployment rolling update configuration parameters\n ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy\n ##\n updateStrategy:\n type: RollingUpdate\n rollingUpdate: {}\n ## @param web.sidecars Add additional sidecar containers to the Airflow web pods\n ## Example:\n ## sidecars:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n sidecars: []\n ## @param web.initContainers Add additional init containers to the Airflow web pods\n ## Example:\n ## initContainers:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n initContainers: []\n ## @param web.extraVolumeMounts Optionally specify extra list of additional volumeMounts for the Airflow web pods\n ##\n extraVolumeMounts: []\n ## @param web.extraVolumes Optionally specify extra list of additional volumes for the Airflow web pods\n ##\n extraVolumes: []\n ## Airflow web Pod Disruption Budget\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/\n ## @param web.pdb.create Deploy a pdb object for the Airflow web pods\n ## @param web.pdb.minAvailable Maximum number/percentage of unavailable Airflow web replicas\n ## @param web.pdb.maxUnavailable Maximum number/percentage of unavailable Airflow web replicas\n ##\n pdb:\n create: false\n minAvailable: 1\n maxUnavailable: \"\"\n\n## @section Airflow scheduler parameters\n\nscheduler:\n ## Bitnami Airflow Scheduler image version\n ## ref: https://hub.docker.com/r/bitnami/airflow-scheduler/tags/\n ## @param scheduler.image.registry Airflow Scheduler image registry\n ## @param scheduler.image.repository Airflow Scheduler image repository\n ## @param scheduler.image.tag Airflow Scheduler image tag (immutable tags are recommended)\n ## @param scheduler.image.digest Airflow Schefuler image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param scheduler.image.pullPolicy Airflow Scheduler image pull policy\n ## @param scheduler.image.pullSecrets Airflow Scheduler image pull secrets\n ## @param scheduler.image.debug Enable image debug mode\n ##\n image:\n registry: docker.io\n repository: bitnami/airflow-scheduler\n tag: 2.4.2-debian-11-r4\n digest: \"\"\n ## Specify a imagePullPolicy\n ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'\n ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images\n ##\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## Set to true if you would like to see extra information on logs\n ##\n debug: false\n ## @param scheduler.replicaCount Number of scheduler replicas\n ##\n replicaCount: 1\n ## @param scheduler.command Override cmd\n ##\n command: []\n ## @param scheduler.args Override args\n ##\n args: []\n ## @param scheduler.extraEnvVars Add extra environment variables\n ##\n extraEnvVars: []\n ## @param scheduler.extraEnvVarsCM ConfigMap with extra environment variables\n ##\n extraEnvVarsCM: \"\"\n ## @param scheduler.extraEnvVarsSecret Secret with extra environment variables\n ##\n extraEnvVarsSecret: \"\"\n ## @param scheduler.extraEnvVarsSecrets List of secrets with extra environment variables for Airflow scheduler pods\n ##\n extraEnvVarsSecrets: []\n ## @param scheduler.customLivenessProbe Custom livenessProbe that overrides the default one\n ##\n customLivenessProbe: {}\n ## @param scheduler.customReadinessProbe Custom readinessProbe that overrides the default one\n ##\n customReadinessProbe: {}\n ## @param scheduler.customStartupProbe Custom startupProbe that overrides the default one\n ##\n customStartupProbe: {}\n ## Airflow scheduler resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ## @param scheduler.resources.limits The resources limits for the Airflow scheduler containers\n ## @param scheduler.resources.requests The requested resources for the Airflow scheduler containers\n ##\n resources:\n limits: {}\n requests: {}\n ## Configure Airflow scheduler pods Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod\n ## @param scheduler.podSecurityContext.enabled Enabled Airflow scheduler pods' Security Context\n ## @param scheduler.podSecurityContext.fsGroup Set Airflow scheduler pod's Security Context fsGroup\n ##\n podSecurityContext:\n enabled: true\n fsGroup: 1001\n ## Configure Airflow scheduler containers (only main one) Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container\n ## @param scheduler.containerSecurityContext.enabled Enabled Airflow scheduler containers' Security Context\n ## @param scheduler.containerSecurityContext.runAsUser Set Airflow scheduler containers' Security Context runAsUser\n ## @param scheduler.containerSecurityContext.runAsNonRoot Set Airflow scheduler containers' Security Context runAsNonRoot\n ##\n containerSecurityContext:\n enabled: true\n runAsUser: 1001\n runAsNonRoot: true\n ## @param scheduler.lifecycleHooks for the Airflow scheduler container(s) to automate configuration before or after startup\n ##\n lifecycleHooks: {}\n ## @param scheduler.hostAliases Deployment pod host aliases\n ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/\n ##\n hostAliases: []\n ## @param scheduler.podLabels Add extra labels to the Airflow scheduler pods\n ##\n podLabels: {}\n ## @param scheduler.podAnnotations Add extra annotations to the Airflow scheduler pods\n ##\n podAnnotations: {}\n ## @param scheduler.affinity Affinity for Airflow scheduler pods assignment (evaluated as a template)\n ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity\n ## Note: `scheduler.podAffinityPreset`, `scheduler.podAntiAffinityPreset`, and `scheduler.nodeAffinityPreset` will be ignored when it's set\n ##\n affinity: {}\n ## Node affinity preset\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity\n ## @param scheduler.nodeAffinityPreset.key Node label key to match. Ignored if `scheduler.affinity` is set.\n ## @param scheduler.nodeAffinityPreset.type Node affinity preset type. Ignored if `scheduler.affinity` is set. Allowed values: `soft` or `hard`\n ## @param scheduler.nodeAffinityPreset.values Node label values to match. Ignored if `scheduler.affinity` is set.\n ##\n nodeAffinityPreset:\n ## e.g:\n ## key: \"kubernetes.io/e2e-az-name\"\n ##\n key: \"\"\n type: \"\"\n ## e.g:\n ## values:\n ## - e2e-az1\n ## - e2e-az2\n ##\n values: []\n ## @param scheduler.nodeSelector Node labels for Airflow scheduler pods assignment\n ## Ref: https://kubernetes.io/docs/user-guide/node-selection/\n ##\n nodeSelector: {}\n ## @param scheduler.podAffinityPreset Pod affinity preset. Ignored if `scheduler.affinity` is set. Allowed values: `soft` or `hard`.\n ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAffinityPreset: \"\"\n ## @param scheduler.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `scheduler.affinity` is set. Allowed values: `soft` or `hard`.\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAntiAffinityPreset: soft\n ## @param scheduler.tolerations Tolerations for Airflow scheduler pods assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n ##\n tolerations: []\n ## @param scheduler.topologySpreadConstraints Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template\n ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods\n ##\n topologySpreadConstraints: []\n ## @param scheduler.priorityClassName Priority Class Name\n ## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass\n ##\n priorityClassName: \"\"\n ## @param scheduler.schedulerName Use an alternate scheduler, e.g. \"stork\".\n ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/\n ##\n schedulerName: \"\"\n ## @param scheduler.terminationGracePeriodSeconds Seconds Airflow scheduler pod needs to terminate gracefully\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods\n ##\n terminationGracePeriodSeconds: \"\"\n ## @param scheduler.updateStrategy.type Airflow scheduler deployment strategy type\n ## @param scheduler.updateStrategy.rollingUpdate Airflow scheduler deployment rolling update configuration parameters\n ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy\n ##\n updateStrategy:\n type: RollingUpdate\n rollingUpdate: {}\n ## @param scheduler.sidecars Add additional sidecar containers to the Airflow scheduler pods\n ## Example:\n ## sidecars:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n sidecars: []\n ## @param scheduler.initContainers Add additional init containers to the Airflow scheduler pods\n ## Example:\n ## initContainers:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n initContainers: []\n ## @param scheduler.extraVolumeMounts Optionally specify extra list of additional volumeMounts for the Airflow scheduler pods\n ##\n extraVolumeMounts: []\n ## @param scheduler.extraVolumes Optionally specify extra list of additional volumes for the Airflow scheduler pods\n ##\n extraVolumes: []\n ## Airflow scheduler Pod Disruption Budget\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/\n ## @param scheduler.pdb.create Deploy a pdb object for the Airflow scheduler pods\n ## @param scheduler.pdb.minAvailable Maximum number/percentage of unavailable Airflow scheduler replicas\n ## @param scheduler.pdb.maxUnavailable Maximum number/percentage of unavailable Airflow scheduler replicas\n ##\n pdb:\n create: false\n minAvailable: 1\n maxUnavailable: \"\"\n\n## @section Airflow worker parameters\n\nworker:\n ## Bitnami Airflow Worker image version\n ## ref: https://hub.docker.com/r/bitnami/airflow-worker/tags/\n ## @param worker.image.registry Airflow Worker image registry\n ## @param worker.image.repository Airflow Worker image repository\n ## @param worker.image.tag Airflow Worker image tag (immutable tags are recommended)\n ## @param worker.image.digest Airflow Worker image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param worker.image.pullPolicy Airflow Worker image pull policy\n ## @param worker.image.pullSecrets Airflow Worker image pull secrets\n ## @param worker.image.debug Enable image debug mode\n ##\n image:\n registry: docker.io\n repository: bitnami/airflow-worker\n tag: 2.4.2-debian-11-r4\n digest: \"\"\n ## Specify a imagePullPolicy\n ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'\n ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images\n ##\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## Set to true if you would like to see extra information on logs\n ##\n debug: false\n ## @param worker.command Override default container command (useful when using custom images)\n ##\n command: []\n ## @param worker.args Override default container args (useful when using custom images)\n ##\n args: []\n ## @param worker.extraEnvVars Array with extra environment variables to add Airflow worker pods\n ##\n extraEnvVars: []\n ## @param worker.extraEnvVarsCM ConfigMap containing extra environment variables for Airflow worker pods\n ##\n extraEnvVarsCM: \"\"\n ## @param worker.extraEnvVarsSecret Secret containing extra environment variables (in case of sensitive data) for Airflow worker pods\n ##\n extraEnvVarsSecret: \"\"\n ## @param worker.extraEnvVarsSecrets List of secrets with extra environment variables for Airflow worker pods\n ##\n extraEnvVarsSecrets: []\n ## @param worker.containerPorts.http Airflow worker HTTP container port\n ##\n containerPorts:\n http: 8793\n ## @param worker.replicaCount Number of Airflow worker replicas\n ##\n replicaCount: 1\n ## Configure extra options for Airflow worker containers' liveness, readiness and startup probes\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes\n ## @param worker.livenessProbe.enabled Enable livenessProbe on Airflow worker containers\n ## @param worker.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe\n ## @param worker.livenessProbe.periodSeconds Period seconds for livenessProbe\n ## @param worker.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe\n ## @param worker.livenessProbe.failureThreshold Failure threshold for livenessProbe\n ## @param worker.livenessProbe.successThreshold Success threshold for livenessProbe\n ##\n livenessProbe:\n enabled: true\n initialDelaySeconds: 180\n periodSeconds: 20\n timeoutSeconds: 5\n failureThreshold: 6\n successThreshold: 1\n ## @param worker.readinessProbe.enabled Enable readinessProbe on Airflow worker containers\n ## @param worker.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe\n ## @param worker.readinessProbe.periodSeconds Period seconds for readinessProbe\n ## @param worker.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe\n ## @param worker.readinessProbe.failureThreshold Failure threshold for readinessProbe\n ## @param worker.readinessProbe.successThreshold Success threshold for readinessProbe\n ##\n readinessProbe:\n enabled: true\n initialDelaySeconds: 30\n periodSeconds: 10\n timeoutSeconds: 5\n failureThreshold: 6\n successThreshold: 1\n ## @param worker.startupProbe.enabled Enable startupProbe on Airflow worker containers\n ## @param worker.startupProbe.initialDelaySeconds Initial delay seconds for startupProbe\n ## @param worker.startupProbe.periodSeconds Period seconds for startupProbe\n ## @param worker.startupProbe.timeoutSeconds Timeout seconds for startupProbe\n ## @param worker.startupProbe.failureThreshold Failure threshold for startupProbe\n ## @param worker.startupProbe.successThreshold Success threshold for startupProbe\n ##\n startupProbe:\n enabled: false\n initialDelaySeconds: 60\n periodSeconds: 10\n timeoutSeconds: 1\n failureThreshold: 15\n successThreshold: 1\n ## @param worker.customLivenessProbe Custom livenessProbe that overrides the default one\n ##\n customLivenessProbe: {}\n ## @param worker.customReadinessProbe Custom readinessProbe that overrides the default one\n ##\n customReadinessProbe: {}\n ## @param worker.customStartupProbe Custom startupProbe that overrides the default one\n ##\n customStartupProbe: {}\n ## Airflow worker resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ## @param worker.resources.limits The resources limits for the Airflow worker containers\n ## @param worker.resources.requests The requested resources for the Airflow worker containers\n ##\n resources:\n limits: {}\n requests: {}\n ## Configure Airflow worker pods Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod\n ## @param worker.podSecurityContext.enabled Enabled Airflow worker pods' Security Context\n ## @param worker.podSecurityContext.fsGroup Set Airflow worker pod's Security Context fsGroup\n ##\n podSecurityContext:\n enabled: true\n fsGroup: 1001\n ## Configure Airflow worker containers (only main one) Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container\n ## @param worker.containerSecurityContext.enabled Enabled Airflow worker containers' Security Context\n ## @param worker.containerSecurityContext.runAsUser Set Airflow worker containers' Security Context runAsUser\n ## @param worker.containerSecurityContext.runAsNonRoot Set Airflow worker containers' Security Context runAsNonRoot\n ##\n containerSecurityContext:\n enabled: true\n runAsUser: 1001\n runAsNonRoot: true\n ## @param worker.lifecycleHooks for the Airflow worker container(s) to automate configuration before or after startup\n ##\n lifecycleHooks: {}\n ## @param worker.hostAliases Deployment pod host aliases\n ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/\n ##\n hostAliases: []\n ## @param worker.podLabels Add extra labels to the Airflow worker pods\n ##\n podLabels: {}\n ## @param worker.podAnnotations Add extra annotations to the Airflow worker pods\n ##\n podAnnotations: {}\n ## @param worker.affinity Affinity for Airflow worker pods assignment (evaluated as a template)\n ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity\n ## Note: `worker.podAffinityPreset`, `worker.podAntiAffinityPreset`, and `worker.nodeAffinityPreset` will be ignored when it's set\n ##\n affinity: {}\n ## Node affinity preset\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity\n ## @param worker.nodeAffinityPreset.key Node label key to match. Ignored if `worker.affinity` is set.\n ## @param worker.nodeAffinityPreset.type Node affinity preset type. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard`\n ## @param worker.nodeAffinityPreset.values Node label values to match. Ignored if `worker.affinity` is set.\n ##\n nodeAffinityPreset:\n ## e.g:\n ## key: \"kubernetes.io/e2e-az-name\"\n ##\n key: \"\"\n type: \"\"\n ## e.g:\n ## values:\n ## - e2e-az1\n ## - e2e-az2\n ##\n values: []\n ## @param worker.nodeSelector Node labels for Airflow worker pods assignment\n ## Ref: https://kubernetes.io/docs/user-guide/node-selection/\n ##\n nodeSelector: {}\n ## @param worker.podAffinityPreset Pod affinity preset. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard`.\n ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAffinityPreset: \"\"\n ## @param worker.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard`.\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAntiAffinityPreset: soft\n ## @param worker.tolerations Tolerations for Airflow worker pods assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n ##\n tolerations: []\n ## @param worker.topologySpreadConstraints Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template\n ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods\n ##\n topologySpreadConstraints: []\n ## @param worker.priorityClassName Priority Class Name\n ## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass\n ##\n priorityClassName: \"\"\n ## @param worker.schedulerName Use an alternate scheduler, e.g. \"stork\".\n ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/\n ##\n schedulerName: \"\"\n ## @param worker.terminationGracePeriodSeconds Seconds Airflow worker pod needs to terminate gracefully\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods\n ##\n terminationGracePeriodSeconds: \"\"\n ## @param worker.updateStrategy.type Airflow worker deployment strategy type\n ## @param worker.updateStrategy.rollingUpdate Airflow worker deployment rolling update configuration parameters\n ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy\n ##\n updateStrategy:\n type: RollingUpdate\n rollingUpdate: {}\n ## @param worker.sidecars Add additional sidecar containers to the Airflow worker pods\n ## Example:\n ## sidecars:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n sidecars: []\n ## @param worker.initContainers Add additional init containers to the Airflow worker pods\n ## Example:\n ## initContainers:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n initContainers: []\n ## @param worker.extraVolumeMounts Optionally specify extra list of additional volumeMounts for the Airflow worker pods\n ##\n extraVolumeMounts: []\n ## @param worker.extraVolumes Optionally specify extra list of additional volumes for the Airflow worker pods\n ##\n extraVolumes: []\n ## @param worker.extraVolumeClaimTemplates Optionally specify extra list of volumesClaimTemplates for the Airflow worker statefulset\n ##\n extraVolumeClaimTemplates: []\n ## @param worker.podTemplate Template to replace the default one to be use when `executor=KubernetesExecutor` to create Airflow worker pods\n ##\n podTemplate: {}\n ## Airflow worker Pod Disruption Budget\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/\n ## @param worker.pdb.create Deploy a pdb object for the Airflow worker pods\n ## @param worker.pdb.minAvailable Maximum number/percentage of unavailable Airflow worker replicas\n ## @param worker.pdb.maxUnavailable Maximum number/percentage of unavailable Airflow worker replicas\n ##\n pdb:\n create: false\n minAvailable: 1\n maxUnavailable: \"\"\n ## Enable HorizontalPodAutoscaler for worker pods\n ## ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/\n ## @param worker.autoscaling.enabled Whether enable horizontal pod autoscaler\n ## @param worker.autoscaling.minReplicas Configure a minimum amount of pods\n ## @param worker.autoscaling.maxReplicas Configure a maximum amount of pods\n ## @param worker.autoscaling.targetCPU Define the CPU target to trigger the scaling actions (utilization percentage)\n ## @param worker.autoscaling.targetMemory Define the memory target to trigger the scaling actions (utilization percentage)\n ##\n autoscaling:\n enabled: false\n minReplicas: 1\n maxReplicas: 3\n targetCPU: 80\n targetMemory: 80\n\n## @section Airflow git sync parameters\n\n## Configure Git to pull dags and plugins\n##\ngit:\n ## Bitnami Git image version\n ## ref: https://hub.docker.com/r/bitnami/git/tags/\n ## @param git.image.registry Git image registry\n ## @param git.image.repository Git image repository\n ## @param git.image.tag Git image tag (immutable tags are recommended)\n ## @param git.image.digest Git image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param git.image.pullPolicy Git image pull policy\n ## @param git.image.pullSecrets Git image pull secrets\n ##\n image:\n registry: docker.io\n repository: bitnami/git\n tag: 2.38.1-debian-11-r7\n digest: \"\"\n ## Specify a imagePullPolicy\n ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'\n ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images\n ##\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## Get DAG files from git repositories\n ## @param git.dags.enabled Enable in order to download DAG files from git repositories.\n ## @param git.dags.repositories [array] Array of repositories from which to download DAG files\n ##\n dags:\n enabled: false\n ## Name for repositories can be anything unique and must follow same naming conventions as kubernetes.\n ## Kubernetes resources can have names up to 253 characters long. The characters allowed in names are:\n ## digits (0-9), lower case letters (a-z), -, and .\n ## Example:\n ## - repository: https://github.com/myuser/myrepo\n ## branch: main\n ## name: my-dags\n ## path: /\n ##\n repositories:\n - repository: \"\"\n ## Branch from repository to checkout\n ##\n branch: \"\"\n ## An unique identifier for repository, must be unique for each repository\n ##\n name: \"\"\n ## Path to a folder in the repository containing the dags\n ##\n path: \"\"\n ## Get Plugins files from git repositories.\n ## @param git.plugins.enabled Enable in order to download Plugins files from git repositories.\n ## @param git.plugins.repositories [array] Array of repositories from which to download DAG files\n ##\n plugins:\n enabled: false\n repositories:\n - repository: \"\"\n ## Branch from repository to checkout\n ##\n branch: \"\"\n ## An unique identifier for repository, must be unique for each repository\n ##\n name: \"\"\n ## Path to a folder in the repository containing the plugins\n ##\n path: \"\"\n ## Properties for the Clone init container\n ## @param git.clone.command Override cmd\n ## @param git.clone.args Override args\n ## @param git.clone.extraVolumeMounts Add extra volume mounts\n ## @param git.clone.extraEnvVars Add extra environment variables\n ## @param git.clone.extraEnvVarsCM ConfigMap with extra environment variables\n ## @param git.clone.extraEnvVarsSecret Secret with extra environment variables\n ## @param git.clone.resources Clone init container resource requests and limits\n ##\n clone:\n command: []\n args: []\n extraVolumeMounts: []\n extraEnvVars: []\n extraEnvVarsCM: \"\"\n extraEnvVarsSecret: \"\"\n ## Clone init container resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ##\n resources: {}\n ## Properties for the Sync sidecar container\n ## @param git.sync.interval Interval in seconds to pull the git repository containing the plugins and/or DAG files\n ## @param git.sync.command Override cmd\n ## @param git.sync.args Override args\n ## @param git.sync.extraVolumeMounts Add extra volume mounts\n ## @param git.sync.extraEnvVars Add extra environment variables\n ## @param git.sync.extraEnvVarsCM ConfigMap with extra environment variables\n ## @param git.sync.extraEnvVarsSecret Secret with extra environment variables\n ## @param git.sync.resources Sync sidecar container resource requests and limits\n ##\n sync:\n interval: 60\n command: []\n args: []\n extraVolumeMounts: []\n extraEnvVars: []\n extraEnvVarsCM: \"\"\n extraEnvVarsSecret: \"\"\n ## Sync sidecar container resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ##\n resources: {}\n\n## @section Airflow ldap parameters\n\n## LDAP configuration\n## @param ldap.enabled Enable LDAP authentication\n## @param ldap.uri Server URI, eg. ldap://ldap_server:389\n## DEPRECATED ldap.base It will be removed in a future release, please use 'ldap.basedn' instead\n## @param ldap.basedn Base of the search, eg. ou=example,o=org.\n## DEPRECATED ldap.uidField It will be removed in a future release,, please use 'ldap.searchAttribute' instead\n## @param ldap.searchAttribute if doing an indirect bind to ldap, this is the field that matches the username when searching for the account to bind to\n## @param ldap.binddn DN of the account used to search in the LDAP server.\n## @param ldap.bindpw Bind Password\n## @param ldap.userRegistration Set to True to enable user self registration\n## @param ldap.userRegistrationRole Set role name to be assign when a user registers himself. This role must already exist. Mandatory when using ldap.userRegistration\n## @param ldap.rolesMapping mapping from LDAP DN to a list of roles\n## @param ldap.rolesSyncAtLogin replace ALL the user's roles each login, or only on registration\n##\nldap:\n enabled: false\n uri: \"ldap://ldap_server:389\"\n basedn: \"dc=example,dc=org\"\n searchAttribute: \"cn\"\n binddn: \"cn=admin,dc=example,dc=org\"\n bindpw: \"\"\n userRegistration: 'True'\n userRegistrationRole: \"Public\"\n rolesMapping: '{ \"cn=All,ou=Groups,dc=example,dc=org\": [\"User\"], \"cn=Admins,ou=Groups,dc=example,dc=org\": [\"Admin\"], }'\n rolesSyncAtLogin: 'True'\n\n ## SSL/TLS parameters for LDAP\n ## @param ldap.tls.enabled Enabled TLS/SSL for LDAP, you must include the CA file.\n ## @param ldap.tls.allowSelfSigned Allow to use self signed certificates\n ## DEPRECATED ldap.tls.CAcertificateSecret It will be removed in a future release, please use ldap.tls.certificatesSecret instead\n ## @param ldap.tls.certificatesSecret Name of the existing secret containing the certificate CA file that will be used by ldap client\n ## @param ldap.tls.certificatesMountPath Where LDAP certifcates are mounted.\n ## DEPRECATED ldap.tls.CAcertificateFilename It will be removed in a future release, please use ldap.tls.CAFilename instead\n ## @param ldap.tls.CAFilename LDAP CA cert filename\n ##\n tls:\n enabled: false\n allowSelfSigned: true\n certificatesSecret: \"\"\n certificatesMountPath: /opt/bitnami/airflow/conf/certs\n CAFilename: \"\"\n\n## @section Traffic Exposure Parameters\n\n## Airflow service parameters\n##\nservice:\n ## @param service.type Airflow service type\n ##\n type: ClusterIP\n ## @param service.ports.http Airflow service HTTP port\n ##\n ports:\n http: 8080\n ## Node ports to expose\n ## @param service.nodePorts.http Node port for HTTP\n ## NOTE: choose port between <30000-32767>\n ##\n nodePorts:\n http: \"\"\n ## @param service.sessionAffinity Control where client requests go, to the same pod or round-robin\n ## Values: ClientIP or None\n ## ref: https://kubernetes.io/docs/user-guide/services/\n ##\n sessionAffinity: None\n ## @param service.sessionAffinityConfig Additional settings for the sessionAffinity\n ## sessionAffinityConfig:\n ## clientIP:\n ## timeoutSeconds: 300\n ##\n sessionAffinityConfig: {}\n ## @param service.clusterIP Airflow service Cluster IP\n ## e.g.:\n ## clusterIP: None\n ##\n clusterIP: \"\"\n ## @param service.loadBalancerIP Airflow service Load Balancer IP\n ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-loadbalancer\n ##\n loadBalancerIP: \"\"\n ## @param service.loadBalancerSourceRanges Airflow service Load Balancer sources\n ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service\n ## e.g:\n ## loadBalancerSourceRanges:\n ## - 10.10.10.0/24\n ##\n loadBalancerSourceRanges: []\n ## @param service.externalTrafficPolicy Airflow service external traffic policy\n ## ref https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip\n ##\n externalTrafficPolicy: Cluster\n ## @param service.annotations Additional custom annotations for Airflow service\n ##\n annotations: {}\n ## @param service.extraPorts Extra port to expose on Airflow service\n ##\n extraPorts: []\n\n## Airflow ingress parameters\n## ref: https://kubernetes.io/docs/user-guide/ingress/\n##\ningress:\n ## @param ingress.enabled Enable ingress record generation for Airflow\n ##\n enabled: false\n ## @param ingress.ingressClassName IngressClass that will be be used to implement the Ingress (Kubernetes 1.18+)\n ## This is supported in Kubernetes 1.18+ and required if you have more than one IngressClass marked as the default for your cluster .\n ## ref: https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/\n ##\n ingressClassName: \"\"\n ## @param ingress.pathType Ingress path type\n ##\n pathType: ImplementationSpecific\n ## @param ingress.apiVersion Force Ingress API version (automatically detected if not set)\n ##\n apiVersion: \"\"\n ## @param ingress.hostname Default host for the ingress record\n ##\n hostname: airflow.local\n ## @param ingress.path Default path for the ingress record\n ## NOTE: You may need to set this to '/*' in order to use this with ALB ingress controllers\n ##\n path: /\n ## @param ingress.annotations [object] Additional annotations for the Ingress resource. To enable certificate autogeneration, place here your cert-manager annotations.\n ## Use this parameter to set the required annotations for cert-manager, see\n ## ref: https://cert-manager.io/docs/usage/ingress/#supported-annotations\n ## e.g:\n ## annotations:\n ## kubernetes.io/ingress.class: nginx\n ## cert-manager.io/cluster-issuer: cluster-issuer-name\n ##\n annotations: {}\n ## @param ingress.tls Enable TLS configuration for the host defined at `ingress.hostname` parameter\n ## TLS certificates will be retrieved from a TLS secret with name: `{{- printf \"%s-tls\" .Values.ingress.hostname }}`\n ## You can:\n ## - Use the `ingress.secrets` parameter to create this TLS secret\n ## - Rely on cert-manager to create it by setting the corresponding annotations\n ## - Rely on Helm to create self-signed certificates by setting `ingress.selfSigned=true`\n ##\n tls: false\n ## @param ingress.selfSigned Create a TLS secret for this ingress record using self-signed certificates generated by Helm\n ##\n selfSigned: false\n ## @param ingress.extraHosts An array with additional hostname(s) to be covered with the ingress record\n ## e.g:\n ## extraHosts:\n ## - name: airflow.local\n ## path: /\n ##\n extraHosts: []\n ## @param ingress.extraPaths An array with additional arbitrary paths that may need to be added to the ingress under the main host\n ## e.g:\n ## extraPaths:\n ## - path: /*\n ## backend:\n ## serviceName: ssl-redirect\n ## servicePort: use-annotation\n ##\n extraPaths: []\n ## @param ingress.extraTls TLS configuration for additional hostname(s) to be covered with this ingress record\n ## ref: https://kubernetes.io/docs/concepts/services-networking/ingress/#tls\n ## e.g:\n ## extraTls:\n ## - hosts:\n ## - airflow.local\n ## secretName: airflow.local-tls\n ##\n extraTls: []\n ## @param ingress.secrets Custom TLS certificates as secrets\n ## NOTE: 'key' and 'certificate' are expected in PEM format\n ## NOTE: 'name' should line up with a 'secretName' set further up\n ## If it is not set and you're using cert-manager, this is unneeded, as it will create a secret for you with valid certificates\n ## If it is not set and you're NOT using cert-manager either, self-signed certificates will be created valid for 365 days\n ## It is also possible to create and manage the certificates outside of this helm chart\n ## Please see README.md for more information\n ## e.g:\n ## secrets:\n ## - name: airflow.local-tls\n ## key: |-\n ## -----BEGIN RSA PRIVATE KEY-----\n ## ...\n ## -----END RSA PRIVATE KEY-----\n ## certificate: |-\n ## -----BEGIN CERTIFICATE-----\n ## ...\n ## -----END CERTIFICATE-----\n ##\n secrets: []\n ## @param ingress.extraRules Additional rules to be covered with this ingress record\n ## ref: https://kubernetes.io/docs/concepts/services-networking/ingress/#ingress-rules\n ## e.g:\n ## extraRules:\n ## - host: example.local\n ## http:\n ## path: /\n ## backend:\n ## service:\n ## name: example-svc\n ## port:\n ## name: http\n ##\n extraRules: []\n\n## @section Other Parameters\n\n## Service account for Airflow pods to use.\n## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/\n##\nserviceAccount:\n ## @param serviceAccount.create Enable creation of ServiceAccount for Airflow pods\n ##\n create: false\n ## @param serviceAccount.name The name of the ServiceAccount to use.\n ## If not set and create is true, a name is generated using the common.names.fullname template\n ##\n name: \"\"\n ## @param serviceAccount.automountServiceAccountToken Allows auto mount of ServiceAccountToken on the serviceAccount created\n ## Can be set to false if pods using this serviceAccount do not need to use K8s API\n ##\n automountServiceAccountToken: true\n ## @param serviceAccount.annotations Additional custom annotations for the ServiceAccount\n ##\n annotations: {}\n## Role Based Access\n## Ref: https://kubernetes.io/docs/admin/authorization/rbac/\n## @param rbac.create Create Role and RoleBinding\n##\nrbac:\n create: false\n ## @param rbac.rules Custom RBAC rules to set\n ## e.g:\n ## rules:\n ## - apiGroups:\n ## - \"\"\n ## resources:\n ## - pods\n ## verbs:\n ## - get\n ## - list\n ##\n rules: []\n\n## @section Airflow metrics parameters\n\nmetrics:\n ## @param metrics.enabled Whether or not to create a standalone Airflow exporter to expose Airflow metrics\n ##\n enabled: false\n ## Bitnami Airflow exporter image\n ## ref: https://hub.docker.com/r/bitnami/airflow-exporter/tags/\n ## @param metrics.image.registry Airflow exporter image registry\n ## @param metrics.image.repository Airflow exporter image repository\n ## @param metrics.image.tag Airflow exporter image tag (immutable tags are recommended)\n ## @param metrics.image.digest Airflow exporter image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param metrics.image.pullPolicy Airflow exporter image pull policy\n ## @param metrics.image.pullSecrets Airflow exporter image pull secrets\n ##\n image:\n registry: docker.io\n repository: bitnami/airflow-exporter\n tag: 0.20220314.0-debian-11-r57\n digest: \"\"\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## @param metrics.extraEnvVars Array with extra environment variables to add Airflow exporter pods\n ##\n extraEnvVars: []\n ## @param metrics.extraEnvVarsCM ConfigMap containing extra environment variables for Airflow exporter pods\n ##\n extraEnvVarsCM: \"\"\n ## @param metrics.extraEnvVarsSecret Secret containing extra environment variables (in case of sensitive data) for Airflow exporter pods\n ##\n extraEnvVarsSecret: \"\"\n ## @param metrics.containerPorts.http Airflow exporter metrics container port\n ##\n containerPorts:\n http: 9112\n ## Airflow exporter resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ## @param metrics.resources.limits The resources limits for the container\n ## @param metrics.resources.requests The requested resources for the container\n ##\n resources:\n limits: {}\n requests: {}\n ## Airflow exporter pods' Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod\n ## @param metrics.podSecurityContext.enabled Enable security context for the pods\n ## @param metrics.podSecurityContext.fsGroup Set Airflow exporter pod's Security Context fsGroup\n ##\n podSecurityContext:\n enabled: true\n fsGroup: 1001\n ## Airflow exporter containers' Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container\n ## @param metrics.containerSecurityContext.enabled Enable Airflow exporter containers' Security Context\n ## @param metrics.containerSecurityContext.runAsUser Set Airflow exporter containers' Security Context runAsUser\n ## @param metrics.containerSecurityContext.runAsNonRoot Set Airflow exporter containers' Security Context runAsNonRoot\n ## e.g:\n ## containerSecurityContext:\n ## enabled: true\n ## capabilities:\n ## drop: [\"NET_RAW\"]\n ## readOnlyRootFilesystem: true\n ##\n containerSecurityContext:\n enabled: true\n runAsUser: 1001\n runAsNonRoot: true\n ## @param metrics.lifecycleHooks for the Airflow exporter container(s) to automate configuration before or after startup\n ##\n lifecycleHooks: {}\n ## @param metrics.hostAliases Airflow exporter pods host aliases\n ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/\n ##\n hostAliases: []\n ## @param metrics.podLabels Extra labels for Airflow exporter pods\n ## Ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/\n ##\n podLabels: {}\n ## @param metrics.podAnnotations Extra annotations for Airflow exporter pods\n ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/\n ##\n podAnnotations: {}\n ## @param metrics.podAffinityPreset Pod affinity preset. Ignored if `metrics.affinity` is set. Allowed values: `soft` or `hard`\n ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAffinityPreset: \"\"\n ## @param metrics.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `metrics.affinity` is set. Allowed values: `soft` or `hard`\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAntiAffinityPreset: soft\n ## Node metrics.affinity preset\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity\n ##\n nodeAffinityPreset:\n ## @param metrics.nodeAffinityPreset.type Node affinity preset type. Ignored if `metrics.affinity` is set. Allowed values: `soft` or `hard`\n ##\n type: \"\"\n ## @param metrics.nodeAffinityPreset.key Node label key to match Ignored if `metrics.affinity` is set.\n ## E.g.\n ## key: \"kubernetes.io/e2e-az-name\"\n ##\n key: \"\"\n ## @param metrics.nodeAffinityPreset.values Node label values to match. Ignored if `metrics.affinity` is set.\n ## E.g.\n ## values:\n ## - e2e-az1\n ## - e2e-az2\n ##\n values: []\n ## @param metrics.affinity Affinity for pod assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity\n ## Note: metrics.podAffinityPreset, metrics.podAntiAffinityPreset, and metrics.nodeAffinityPreset will be ignored when it's set\n ##\n affinity: {}\n ## @param metrics.nodeSelector Node labels for pod assignment\n ## Ref: https://kubernetes.io/docs/user-guide/node-selection/\n ##\n nodeSelector: {}\n ## @param metrics.tolerations Tolerations for pod assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n ##\n tolerations: []\n ## @param metrics.schedulerName Name of the k8s scheduler (other than default) for Airflow exporter\n ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/\n ##\n schedulerName: \"\"\n ## Airflow exporter service configuration\n ##\n service:\n ## @param metrics.service.ports.http Airflow exporter metrics service port\n ##\n ports:\n http: 9112\n ## @param metrics.service.clusterIP Static clusterIP or None for headless services\n ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#choosing-your-own-ip-address\n ##\n clusterIP: \"\"\n ## @param metrics.service.sessionAffinity Control where client requests go, to the same pod or round-robin\n ## Values: ClientIP or None\n ## ref: https://kubernetes.io/docs/user-guide/services/\n ##\n sessionAffinity: None\n ## @param metrics.service.annotations [object] Annotations for the Airflow exporter service\n ##\n annotations:\n prometheus.io/scrape: \"true\"\n prometheus.io/port: \"{{ .Values.metrics.service.ports.http }}\"\n ## Prometheus Operator ServiceMonitor configuration\n ##\n serviceMonitor:\n ## @param metrics.serviceMonitor.enabled if `true`, creates a Prometheus Operator ServiceMonitor (requires `metrics.enabled` to be `true`)\n ##\n enabled: false\n ## @param metrics.serviceMonitor.namespace Namespace in which Prometheus is running\n ##\n namespace: \"\"\n ## @param metrics.serviceMonitor.interval Interval at which metrics should be scraped\n ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint\n ##\n interval: \"\"\n ## @param metrics.serviceMonitor.scrapeTimeout Timeout after which the scrape is ended\n ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint\n ##\n scrapeTimeout: \"\"\n ## @param metrics.serviceMonitor.labels Additional labels that can be used so ServiceMonitor will be discovered by Prometheus\n ##\n labels: {}\n ## @param metrics.serviceMonitor.selector Prometheus instance selector labels\n ## ref: https://github.com/bitnami/charts/tree/main/bitnami/prometheus-operator#prometheus-configuration\n ##\n selector: {}\n ## @param metrics.serviceMonitor.relabelings RelabelConfigs to apply to samples before scraping\n ##\n relabelings: []\n ## @param metrics.serviceMonitor.metricRelabelings MetricRelabelConfigs to apply to samples before ingestion\n ##\n metricRelabelings: []\n ## @param metrics.serviceMonitor.honorLabels Specify honorLabels parameter to add the scrape endpoint\n ##\n honorLabels: false\n ## @param metrics.serviceMonitor.jobLabel The name of the label on the target service to use as the job name in prometheus.\n ##\n jobLabel: \"\"\n\n## @section Airflow database parameters\n\n## PostgreSQL chart configuration\n## ref: https://github.com/bitnami/charts/blob/main/bitnami/postgresql/values.yaml\n## @param postgresql.enabled Switch to enable or disable the PostgreSQL helm chart\n## @param postgresql.auth.enablePostgresUser Assign a password to the \"postgres\" admin user. Otherwise, git access will be blocked for this user\n## @param postgresql.auth.username Name for a custom user to create\n## @param postgresql.auth.password Password for the custom user to create\n## @param postgresql.auth.database Name for a custom database to create\n## @param postgresql.auth.existingSecret Name of existing secret to use for PostgreSQL credentials\n## @param postgresql.architecture PostgreSQL architecture (`standalone` or `replication`)\n##\npostgresql:\n enabled: true\n auth:\n enablePostgresUser: false\n username: bn_airflow\n password: \"\"\n database: bitnami_airflow\n existingSecret: \"\"\n architecture: standalone\n## External PostgreSQL configuration\n## All of these values are only used when postgresql.enabled is set to false\n## @param externalDatabase.host Database host\n## @param externalDatabase.port Database port number\n## @param externalDatabase.user Non-root username for Airflow\n## @param externalDatabase.password Password for the non-root username for Airflow\n## @param externalDatabase.database Airflow database name\n## @param externalDatabase.existingSecret Name of an existing secret resource containing the database credentials\n## @param externalDatabase.existingSecretPasswordKey Name of an existing secret key containing the database credentials\n##\nexternalDatabase:\n host: localhost\n port: 5432\n user: bn_airflow\n database: bitnami_airflow\n password: \"\"\n existingSecret: \"\"\n existingSecretPasswordKey: \"\"\n\n## Redis® chart configuration\n## ref: https://github.com/bitnami/charts/blob/main/bitnami/redis/values.yaml\n## @param redis.enabled Switch to enable or disable the Redis® helm\n## @param redis.auth.enabled Enable password authentication\n## @param redis.auth.password Redis® password\n## @param redis.auth.existingSecret The name of an existing secret with Redis® credentials\n## @param redis.architecture Redis® architecture. Allowed values: `standalone` or `replication`\n##\nredis:\n enabled: true\n auth:\n enabled: true\n ## Redis® password (both master and slave). Defaults to a random 10-character alphanumeric string if not set and auth.enabled is true.\n ## It should always be set using the password value or in the existingSecret to avoid issues\n ## with Airflow.\n ## The password value is ignored if existingSecret is set\n password: \"\"\n existingSecret: \"\"\n architecture: standalone\n\n## External Redis® configuration\n## All of these values are only used when redis.enabled is set to false\n## @param externalRedis.host Redis® host\n## @param externalRedis.port Redis® port number\n## @param externalRedis.username Redis® username\n## @param externalRedis.password Redis® password\n## @param externalRedis.existingSecret Name of an existing secret resource containing the Redis&trade credentials\n## @param externalRedis.existingSecretPasswordKey Name of an existing secret key containing the Redis&trade credentials\n##\nexternalRedis:\n host: localhost\n port: 6379\n ## Most Redis® implementations do not require a username\n ## to authenticate and it should be enough with the password\n username: \"\"\n password: \"\"\n existingSecret: \"\"\n existingSecretPasswordKey: \"\"\n", Readme: "", UserId: 6, ReferenceValueId: 6304, diff --git a/pkg/appStore/deployment/service/InstalledAppService.go b/pkg/appStore/deployment/service/InstalledAppService.go index bfdd857d1f..4f33452280 100644 --- a/pkg/appStore/deployment/service/InstalledAppService.go +++ b/pkg/appStore/deployment/service/InstalledAppService.go @@ -22,7 +22,7 @@ import ( "context" commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" - "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" /* #nosec */ "crypto/sha1" @@ -124,7 +124,7 @@ type InstalledAppServiceImpl struct { k8sApplicationService application3.K8sApplicationService acdConfig *argocdServer.ACDConfig gitOpsConfigReadService config.GitOpsConfigReadService - gitOpsRemoteOperationService remote.GitOpsRemoteOperationService + gitOpsRemoteOperationService git.GitOpsRemoteOperationService } func NewInstalledAppServiceImpl(logger *zap.SugaredLogger, @@ -137,7 +137,7 @@ func NewInstalledAppServiceImpl(logger *zap.SugaredLogger, pubsubClient *pubsub.PubSubClientServiceImpl, chartGroupDeploymentRepository repository6.ChartGroupDeploymentRepository, envService cluster2.EnvironmentService, - aCDAuthConfig *util2.ACDAuthConfig, gitOpsRepository repository3.GitOpsConfigRepository, userService user.UserService, + aCDAuthConfig *util2.ACDAuthConfig, gitOpsRepository repository3.GitOpsConfigRepository, userService user.UserService, appStoreDeploymentFullModeService appStoreDeploymentFullMode.AppStoreDeploymentFullModeService, appStoreDeploymentService AppStoreDeploymentService, installedAppRepositoryHistory repository2.InstalledAppVersionHistoryRepository, @@ -147,7 +147,7 @@ func NewInstalledAppServiceImpl(logger *zap.SugaredLogger, appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService, k8sCommonService k8s.K8sCommonService, k8sApplicationService application3.K8sApplicationService, acdConfig *argocdServer.ACDConfig, gitOpsConfigReadService config.GitOpsConfigReadService, - gitOpsRemoteOperationService remote.GitOpsRemoteOperationService) (*InstalledAppServiceImpl, error) { + gitOpsRemoteOperationService git.GitOpsRemoteOperationService) (*InstalledAppServiceImpl, error) { impl := &InstalledAppServiceImpl{ logger: logger, installedAppRepository: installedAppRepository, diff --git a/pkg/appStore/deployment/tool/AppStoreDeploymentArgoCdService.go b/pkg/appStore/deployment/tool/AppStoreDeploymentArgoCdService.go index 41c2b080af..f5973894d5 100644 --- a/pkg/appStore/deployment/tool/AppStoreDeploymentArgoCdService.go +++ b/pkg/appStore/deployment/tool/AppStoreDeploymentArgoCdService.go @@ -6,7 +6,7 @@ import ( "errors" "fmt" commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" - "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" "net/http" "strings" "time" @@ -71,7 +71,7 @@ type AppStoreDeploymentArgoCdServiceImpl struct { appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository argoClientWrapperService argocdServer.ArgoClientWrapperService acdConfig *argocdServer.ACDConfig - gitOpsRemoteOperationService remote.GitOpsRemoteOperationService + gitOpsRemoteOperationService git.GitOpsRemoteOperationService } func NewAppStoreDeploymentArgoCdServiceImpl(logger *zap.SugaredLogger, appStoreDeploymentFullModeService appStoreDeploymentFullMode.AppStoreDeploymentFullModeService, @@ -83,7 +83,7 @@ func NewAppStoreDeploymentArgoCdServiceImpl(logger *zap.SugaredLogger, appStoreD pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository, appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository, argoClientWrapperService argocdServer.ArgoClientWrapperService, acdConfig *argocdServer.ACDConfig, - gitOpsRemoteOperationService remote.GitOpsRemoteOperationService) *AppStoreDeploymentArgoCdServiceImpl { + gitOpsRemoteOperationService git.GitOpsRemoteOperationService) *AppStoreDeploymentArgoCdServiceImpl { return &AppStoreDeploymentArgoCdServiceImpl{ Logger: logger, appStoreDeploymentFullModeService: appStoreDeploymentFullModeService, diff --git a/pkg/appStore/deployment/tool/AppStoreDeploymentHelmService.go b/pkg/appStore/deployment/tool/AppStoreDeploymentHelmService.go index e1384780ed..9b42f2bd9f 100644 --- a/pkg/appStore/deployment/tool/AppStoreDeploymentHelmService.go +++ b/pkg/appStore/deployment/tool/AppStoreDeploymentHelmService.go @@ -5,7 +5,7 @@ import ( "errors" repository2 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" - "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" "time" client "github.com/devtron-labs/devtron/api/helm-app" @@ -39,7 +39,7 @@ type AppStoreDeploymentHelmServiceImpl struct { installedAppRepository repository.InstalledAppRepository appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService OCIRegistryConfigRepository repository2.OCIRegistryConfigRepository - gitOpsRemoteOperationService remote.GitOpsRemoteOperationService + gitOpsRemoteOperationService git.GitOpsRemoteOperationService } func NewAppStoreDeploymentHelmServiceImpl(logger *zap.SugaredLogger, helmAppService client.HelmAppService, @@ -47,7 +47,7 @@ func NewAppStoreDeploymentHelmServiceImpl(logger *zap.SugaredLogger, helmAppServ helmAppClient client.HelmAppClient, installedAppRepository repository.InstalledAppRepository, appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService, OCIRegistryConfigRepository repository2.OCIRegistryConfigRepository, - gitOpsRemoteOperationService remote.GitOpsRemoteOperationService) *AppStoreDeploymentHelmServiceImpl { + gitOpsRemoteOperationService git.GitOpsRemoteOperationService) *AppStoreDeploymentHelmServiceImpl { return &AppStoreDeploymentHelmServiceImpl{ Logger: logger, helmAppService: helmAppService, diff --git a/pkg/deployment/gitOps/remote/GitOpsRemoteOperationService.go b/pkg/deployment/gitOps/git/GitOpsRemoteOperationService.go similarity index 99% rename from pkg/deployment/gitOps/remote/GitOpsRemoteOperationService.go rename to pkg/deployment/gitOps/git/GitOpsRemoteOperationService.go index 2501b94be4..375031ad60 100644 --- a/pkg/deployment/gitOps/remote/GitOpsRemoteOperationService.go +++ b/pkg/deployment/gitOps/git/GitOpsRemoteOperationService.go @@ -1,4 +1,4 @@ -package remote +package git import ( "fmt" @@ -6,7 +6,7 @@ import ( "github.com/devtron-labs/devtron/internal/util" commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" - "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote/bean" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git/bean" dirCopy "github.com/otiai10/copy" "go.uber.org/zap" "k8s.io/helm/pkg/proto/hapi/chart" diff --git a/pkg/deployment/gitOps/remote/bean/bean.go b/pkg/deployment/gitOps/git/bean/bean.go similarity index 100% rename from pkg/deployment/gitOps/remote/bean/bean.go rename to pkg/deployment/gitOps/git/bean/bean.go diff --git a/pkg/deployment/gitOps/wire_gitOps.go b/pkg/deployment/gitOps/wire_gitOps.go index 7d5378d238..92a256a4e4 100644 --- a/pkg/deployment/gitOps/wire_gitOps.go +++ b/pkg/deployment/gitOps/wire_gitOps.go @@ -2,7 +2,7 @@ package gitOps import ( "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" - "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" "github.com/google/wire" ) @@ -10,6 +10,6 @@ var GitOpsWireSet = wire.NewSet( config.NewGitOpsConfigReadServiceImpl, wire.Bind(new(config.GitOpsConfigReadService), new(*config.GitOpsConfigReadServiceImpl)), - remote.NewGitOpsRemoteOperationServiceImpl, - wire.Bind(new(remote.GitOpsRemoteOperationService), new(*remote.GitOpsRemoteOperationServiceImpl)), + git.NewGitOpsRemoteOperationServiceImpl, + wire.Bind(new(git.GitOpsRemoteOperationService), new(*git.GitOpsRemoteOperationServiceImpl)), ) diff --git a/pkg/pipeline/DeploymentPipelineConfigService.go b/pkg/pipeline/DeploymentPipelineConfigService.go index 98e9c35512..273145d716 100644 --- a/pkg/pipeline/DeploymentPipelineConfigService.go +++ b/pkg/pipeline/DeploymentPipelineConfigService.go @@ -43,7 +43,7 @@ import ( repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" - "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/history" @@ -148,7 +148,7 @@ type CdPipelineConfigServiceImpl struct { argoClientWrapperService argocdServer.ArgoClientWrapperService deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService gitOpsConfigReadService config.GitOpsConfigReadService - gitOpsRemoteOperationService remote.GitOpsRemoteOperationService + gitOpsRemoteOperationService git.GitOpsRemoteOperationService } func NewCdPipelineConfigServiceImpl(logger *zap.SugaredLogger, pipelineRepository pipelineConfig.PipelineRepository, @@ -169,7 +169,7 @@ func NewCdPipelineConfigServiceImpl(logger *zap.SugaredLogger, pipelineRepositor argoClientWrapperService argocdServer.ArgoClientWrapperService, deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService, gitOpsConfigReadService config.GitOpsConfigReadService, - gitOpsRemoteOperationService remote.GitOpsRemoteOperationService) *CdPipelineConfigServiceImpl { + gitOpsRemoteOperationService git.GitOpsRemoteOperationService) *CdPipelineConfigServiceImpl { return &CdPipelineConfigServiceImpl{ logger: logger, pipelineRepository: pipelineRepository, @@ -1593,7 +1593,7 @@ func (impl *CdPipelineConfigServiceImpl) RegisterInACD(gitOpsRepoName string, ch err := impl.argoClientWrapperService.RegisterGitOpsRepoInArgo(ctx, chartGitAttr.RepoUrl) if err != nil { impl.logger.Errorw("error while register git repo in argo", "err", err) - emptyRepoErrorMessage := []string{"failed to get index: 404 Not Found", "remote repository is empty"} + emptyRepoErrorMessage := []string{"failed to get index: 404 Not Found", "git repository is empty"} if strings.Contains(err.Error(), emptyRepoErrorMessage[0]) || strings.Contains(err.Error(), emptyRepoErrorMessage[1]) { // - found empty repository, create some file in repository err := impl.gitOpsRemoteOperationService.CreateReadmeInGitRepo(gitOpsRepoName, userId) diff --git a/pkg/pipeline/PipelineStageServiceIT_test.go b/pkg/pipeline/PipelineStageServiceIT_test.go index f6e8d803c5..532d509eb7 100644 --- a/pkg/pipeline/PipelineStageServiceIT_test.go +++ b/pkg/pipeline/PipelineStageServiceIT_test.go @@ -26,9 +26,9 @@ import ( "time" ) -const CiPipelineStageCreateReqJson = `{"appId":1,"appWorkflowId":0,"action":0,"ciPipeline":{"active":true,"ciMaterial":[{"gitMaterialId":1,"id":0,"source":{"type":"SOURCE_TYPE_BRANCH_FIXED","value":"main","regex":""}}],"dockerArgs":{},"externalCiConfig":{},"id":0,"isExternal":false,"isManual":false,"name":"ci-1-xyze","linkedCount":0,"scanEnabled":false,"preBuildStage":{"id":0,"steps":[{"id":1,"index":1,"name":"Task 1","description":"chbsdkhbc","stepType":"INLINE","directoryPath":"","inlineStepDetail":{"scriptType":"CONTAINER_IMAGE","script":"echo \"ifudsbvnv\"","conditionDetails":[],"inputVariables":[{"id":1,"name":"Hello","value":"","format":"STRING","description":"jnsdvbdvbsd","defaultValue":"","variableType":"GLOBAL","refVariableStepIndex":0,"refVariableName":"WORKING_DIRECTORY","refVariableStage":""}],"outputVariables":null,"commandArgsMap":[{"command":"echo","args":["\"HOSTNAME\"","\"PORT\""]}],"portMap":[{"portOnLocal":8080,"portOnContainer":9090}],"mountCodeToContainer":true,"mountDirectoryFromHost":true,"mountCodeToContainerPath":"/sourcecode","mountPathMap":[{"filePathOnDisk":"./test","filePathOnContainer":"./test_container"}],"containerImagePath":"python:latest","isMountCustomScript":true,"storeScriptAt":"./directory/script"},"outputDirectoryPath":["./test1"]},{"id":2,"index":2,"name":"K6 Load testing","description":"K6 is an open-source tool and cloud service that makes load testing easy for developers and QA engineers.","stepType":"REF_PLUGIN","directoryPath":"","pluginRefStepDetail":{"id":0,"pluginId":1,"conditionDetails":[{"id":0,"conditionOnVariable":"RelativePathToScript","conditionOperator":"==","conditionType":"TRIGGER","conditionalValue":"svfsv"},{"id":0,"conditionOnVariable":"PrometheusApiKey","conditionOperator":"==","conditionType":"TRIGGER","conditionalValue":"dfbeavafsv"}],"inputVariables":[{"id":1,"name":"RelativePathToScript","format":"STRING","description":"checkout path + script path along with script name","isExposed":true,"allowEmptyValue":false,"defaultValue":"/./script.js","variableType":"NEW","variableStepIndexInPlugin":1,"value":"dethdt","refVariableName":"","refVariableStage":""},{"id":2,"name":"PrometheusUsername","format":"STRING","description":"username of prometheus account","isExposed":true,"allowEmptyValue":true,"defaultValue":"","variableType":"NEW","variableStepIndexInPlugin":1,"value":"ghmfnbd","refVariableName":"","refVariableStage":""},{"id":3,"name":"PrometheusApiKey","format":"STRING","description":"api key of prometheus account","isExposed":true,"allowEmptyValue":true,"defaultValue":"","variableType":"NEW","variableStepIndexInPlugin":1,"value":"afegs","refVariableName":"","refVariableStage":""},{"id":4,"name":"PrometheusRemoteWriteEndpoint","format":"STRING","description":"remote write endpoint of prometheus account","isExposed":true,"allowEmptyValue":true,"defaultValue":"","variableType":"NEW","variableStepIndexInPlugin":1,"value":"aef","refVariableName":"","refVariableStage":""},{"id":5,"name":"OutputType","format":"STRING","description":"output type - LOG or PROMETHEUS","isExposed":true,"allowEmptyValue":false,"defaultValue":"LOG","variableType":"NEW","variableStepIndexInPlugin":1,"value":"fdgn","refVariableName":"","refVariableStage":""}]}},{"id":3,"index":3,"name":"Task 3","description":"sfdbvf","stepType":"INLINE","directoryPath":"","inlineStepDetail":{"scriptType":"SHELL","script":"#!/bin/sh \nset -eo pipefail \n#set -v ## uncomment this to debug the script \n","conditionDetails":[{"id":0,"conditionOnVariable":"Hello","conditionOperator":"==","conditionType":"PASS","conditionalValue":"aedfrwgwr"},{"id":0,"conditionOnVariable":"Hello","conditionOperator":"!=","conditionType":"PASS","conditionalValue":"tegegr"}],"inputVariables":[],"outputVariables":[{"id":1,"name":"Hello","value":"","format":"STRING","description":"dsuihvsuvhbdv","defaultValue":"","variableType":"NEW","refVariableStepIndex":0,"refVariableName":""}],"commandArgsMap":[{"command":"","args":[]}],"portMap":[],"mountCodeToContainer":false,"mountDirectoryFromHost":false},"outputDirectoryPath":["./test2"]}]},"postBuildStage":{},"dockerConfigOverride":{}}}` -const CiPipelineStageUpdateReqJson = `{"appId":1,"appWorkflowId":3,"action":1,"ciPipeline":{"isManual":false,"dockerArgs":{},"isExternal":false,"parentCiPipeline":0,"parentAppId":0,"appId":1,"externalCiConfig":{"id":0,"webhookUrl":"","payload":"","accessKey":"","payloadOption":null,"schema":null,"responses":null,"projectId":0,"projectName":"","environmentId":"","environmentName":"","environmentIdentifier":"","appId":0,"appName":"","role":""},"ciMaterial":[{"gitMaterialId":1,"id":3,"source":{"type":"SOURCE_TYPE_BRANCH_FIXED","value":"main","regex":""}}],"name":"ci-1-unov","id":3,"active":true,"linkedCount":0,"scanEnabled":false,"appWorkflowId":3,"preBuildStage":{"id":5,"type":"PRE_CI","steps":[{"id":9,"name":"K6 Load testing","description":"K6 is an open-source tool and cloud service that makes load testing easy for developers and QA engineers.","index":1,"stepType":"REF_PLUGIN","outputDirectoryPath":null,"inlineStepDetail":null,"pluginRefStepDetail":{"pluginId":1,"inputVariables":[{"id":44,"name":"RelativePathToScript","format":"STRING","description":"checkout path + script path along with script name","isExposed":true,"defaultValue":"/./script.js","value":"sfds","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""},{"id":45,"name":"PrometheusUsername","format":"STRING","description":"username of prometheus account","isExposed":true,"allowEmptyValue":true,"value":"sdf","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""},{"id":46,"name":"PrometheusApiKey","format":"STRING","description":"api key of prometheus account","isExposed":true,"allowEmptyValue":true,"value":"hter","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""},{"id":47,"name":"PrometheusRemoteWriteEndpoint","format":"STRING","description":"remote write endpoint of prometheus account","isExposed":true,"allowEmptyValue":true,"value":"ewq","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""},{"id":48,"name":"OutputType","format":"STRING","description":"output type - LOG or PROMETHEUS","isExposed":true,"defaultValue":"LOG","value":"erg","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""}],"outputVariables":null,"conditionDetails":null},"triggerIfParentStageFail":false},{"id":10,"name":"Task 3","description":"","index":3,"stepType":"INLINE","outputDirectoryPath":["./asap"],"inlineStepDetail":{"scriptType":"CONTAINER_IMAGE","script":null,"storeScriptAt":null,"mountDirectoryFromHost":false,"containerImagePath":"alpine:latest","commandArgsMap":[{"command":"echo","args":["HOSTNAME"]}],"inputVariables":null,"outputVariables":null,"conditionDetails":null,"portMap":[],"mountCodeToContainerPath":null,"mountPathMap":null},"pluginRefStepDetail":null,"triggerIfParentStageFail":false},{"id":8,"name":"Task 1","description":"","index":2,"stepType":"INLINE","outputDirectoryPath":["./test"],"inlineStepDetail":{"scriptType":"SHELL","script":"#!/bin/sh \nset -eo pipefail \necho \"Hello from inside pre-build stage\"\n#set -v ## uncomment this to debug the script \n","storeScriptAt":"","mountDirectoryFromHost":false,"commandArgsMap":[{"command":"","args":null}],"inputVariables":null,"outputVariables":null,"conditionDetails":null},"pluginRefStepDetail":null,"triggerIfParentStageFail":false}]},"postBuildStage":{},"isDockerConfigOverridden":false,"dockerConfigOverride":{}}}` -const CiPipelineStageDeleteReqJson = `{"appId":1,"appWorkflowId":8,"action":2,"ciPipeline":{"isManual":false,"dockerArgs":{},"isExternal":false,"parentCiPipeline":0,"parentAppId":0,"appId":1,"externalCiConfig":{"id":0,"webhookUrl":"","payload":"","accessKey":"","payloadOption":null,"schema":null,"responses":null,"projectId":0,"projectName":"","environmentId":"","environmentName":"","environmentIdentifier":"","appId":0,"appName":"","role":""},"ciMaterial":[{"gitMaterialId":1,"id":8,"source":{"type":"SOURCE_TYPE_BRANCH_FIXED","value":"main","regex":""}}],"name":"ci-1-unjn","id":8,"active":true,"linkedCount":0,"scanEnabled":false,"appWorkflowId":8,"preBuildStage":{"id":7,"type":"PRE_CI","steps":[{"id":11,"name":"Task 1","description":"","index":1,"stepType":"INLINE","outputDirectoryPath":["./test"],"inlineStepDetail":{"scriptType":"SHELL","script":"#!/bin/sh \nset -eo pipefail \necho \"Prakash\"\n#set -v ## uncomment this to debug the script \n","storeScriptAt":"","mountDirectoryFromHost":false,"commandArgsMap":[{"command":"","args":null}],"inputVariables":null,"outputVariables":null,"conditionDetails":null},"pluginRefStepDetail":null,"triggerIfParentStageFail":false},{"id":12,"name":"K6 Load testing","description":"K6 is an open-source tool and cloud service that makes load testing easy for developers and QA engineers.","index":2,"stepType":"REF_PLUGIN","outputDirectoryPath":null,"inlineStepDetail":null,"pluginRefStepDetail":{"pluginId":1,"inputVariables":[{"id":49,"name":"RelativePathToScript","format":"STRING","description":"checkout path + script path along with script name","isExposed":true,"defaultValue":"/./script.js","value":"sfds","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""},{"id":50,"name":"PrometheusUsername","format":"STRING","description":"username of prometheus account","isExposed":true,"allowEmptyValue":true,"value":"sdf","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""},{"id":51,"name":"PrometheusApiKey","format":"STRING","description":"api key of prometheus account","isExposed":true,"allowEmptyValue":true,"value":"hter","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""},{"id":52,"name":"PrometheusRemoteWriteEndpoint","format":"STRING","description":"remote write endpoint of prometheus account","isExposed":true,"allowEmptyValue":true,"value":"ewq","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""},{"id":53,"name":"OutputType","format":"STRING","description":"output type - LOG or PROMETHEUS","isExposed":true,"defaultValue":"LOG","value":"erg","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""}],"outputVariables":null,"conditionDetails":null},"triggerIfParentStageFail":false},{"id":13,"name":"Task 3","description":"","index":3,"stepType":"INLINE","outputDirectoryPath":["./asap"],"inlineStepDetail":{"scriptType":"CONTAINER_IMAGE","script":"","storeScriptAt":"","mountDirectoryFromHost":false,"containerImagePath":"alpine:latest","commandArgsMap":[{"command":"echo","args":["HOSTNAME"]}],"inputVariables":null,"outputVariables":null,"conditionDetails":null,"portMap":[]},"pluginRefStepDetail":null,"triggerIfParentStageFail":false}]},"isDockerConfigOverridden":false,"dockerConfigOverride":{},"postBuildStage":{}}}` +const CiPipelineStageCreateReqJson = `{"appId":1,"appWorkflowId":0,"action":0,"ciPipeline":{"active":true,"ciMaterial":[{"gitMaterialId":1,"id":0,"source":{"type":"SOURCE_TYPE_BRANCH_FIXED","value":"main","regex":""}}],"dockerArgs":{},"externalCiConfig":{},"id":0,"isExternal":false,"isManual":false,"name":"ci-1-xyze","linkedCount":0,"scanEnabled":false,"preBuildStage":{"id":0,"steps":[{"id":1,"index":1,"name":"Task 1","description":"chbsdkhbc","stepType":"INLINE","directoryPath":"","inlineStepDetail":{"scriptType":"CONTAINER_IMAGE","script":"echo \"ifudsbvnv\"","conditionDetails":[],"inputVariables":[{"id":1,"name":"Hello","value":"","format":"STRING","description":"jnsdvbdvbsd","defaultValue":"","variableType":"GLOBAL","refVariableStepIndex":0,"refVariableName":"WORKING_DIRECTORY","refVariableStage":""}],"outputVariables":null,"commandArgsMap":[{"command":"echo","args":["\"HOSTNAME\"","\"PORT\""]}],"portMap":[{"portOnLocal":8080,"portOnContainer":9090}],"mountCodeToContainer":true,"mountDirectoryFromHost":true,"mountCodeToContainerPath":"/sourcecode","mountPathMap":[{"filePathOnDisk":"./test","filePathOnContainer":"./test_container"}],"containerImagePath":"python:latest","isMountCustomScript":true,"storeScriptAt":"./directory/script"},"outputDirectoryPath":["./test1"]},{"id":2,"index":2,"name":"K6 Load testing","description":"K6 is an open-source tool and cloud service that makes load testing easy for developers and QA engineers.","stepType":"REF_PLUGIN","directoryPath":"","pluginRefStepDetail":{"id":0,"pluginId":1,"conditionDetails":[{"id":0,"conditionOnVariable":"RelativePathToScript","conditionOperator":"==","conditionType":"TRIGGER","conditionalValue":"svfsv"},{"id":0,"conditionOnVariable":"PrometheusApiKey","conditionOperator":"==","conditionType":"TRIGGER","conditionalValue":"dfbeavafsv"}],"inputVariables":[{"id":1,"name":"RelativePathToScript","format":"STRING","description":"checkout path + script path along with script name","isExposed":true,"allowEmptyValue":false,"defaultValue":"/./script.js","variableType":"NEW","variableStepIndexInPlugin":1,"value":"dethdt","refVariableName":"","refVariableStage":""},{"id":2,"name":"PrometheusUsername","format":"STRING","description":"username of prometheus account","isExposed":true,"allowEmptyValue":true,"defaultValue":"","variableType":"NEW","variableStepIndexInPlugin":1,"value":"ghmfnbd","refVariableName":"","refVariableStage":""},{"id":3,"name":"PrometheusApiKey","format":"STRING","description":"api key of prometheus account","isExposed":true,"allowEmptyValue":true,"defaultValue":"","variableType":"NEW","variableStepIndexInPlugin":1,"value":"afegs","refVariableName":"","refVariableStage":""},{"id":4,"name":"PrometheusRemoteWriteEndpoint","format":"STRING","description":"git write endpoint of prometheus account","isExposed":true,"allowEmptyValue":true,"defaultValue":"","variableType":"NEW","variableStepIndexInPlugin":1,"value":"aef","refVariableName":"","refVariableStage":""},{"id":5,"name":"OutputType","format":"STRING","description":"output type - LOG or PROMETHEUS","isExposed":true,"allowEmptyValue":false,"defaultValue":"LOG","variableType":"NEW","variableStepIndexInPlugin":1,"value":"fdgn","refVariableName":"","refVariableStage":""}]}},{"id":3,"index":3,"name":"Task 3","description":"sfdbvf","stepType":"INLINE","directoryPath":"","inlineStepDetail":{"scriptType":"SHELL","script":"#!/bin/sh \nset -eo pipefail \n#set -v ## uncomment this to debug the script \n","conditionDetails":[{"id":0,"conditionOnVariable":"Hello","conditionOperator":"==","conditionType":"PASS","conditionalValue":"aedfrwgwr"},{"id":0,"conditionOnVariable":"Hello","conditionOperator":"!=","conditionType":"PASS","conditionalValue":"tegegr"}],"inputVariables":[],"outputVariables":[{"id":1,"name":"Hello","value":"","format":"STRING","description":"dsuihvsuvhbdv","defaultValue":"","variableType":"NEW","refVariableStepIndex":0,"refVariableName":""}],"commandArgsMap":[{"command":"","args":[]}],"portMap":[],"mountCodeToContainer":false,"mountDirectoryFromHost":false},"outputDirectoryPath":["./test2"]}]},"postBuildStage":{},"dockerConfigOverride":{}}}` +const CiPipelineStageUpdateReqJson = `{"appId":1,"appWorkflowId":3,"action":1,"ciPipeline":{"isManual":false,"dockerArgs":{},"isExternal":false,"parentCiPipeline":0,"parentAppId":0,"appId":1,"externalCiConfig":{"id":0,"webhookUrl":"","payload":"","accessKey":"","payloadOption":null,"schema":null,"responses":null,"projectId":0,"projectName":"","environmentId":"","environmentName":"","environmentIdentifier":"","appId":0,"appName":"","role":""},"ciMaterial":[{"gitMaterialId":1,"id":3,"source":{"type":"SOURCE_TYPE_BRANCH_FIXED","value":"main","regex":""}}],"name":"ci-1-unov","id":3,"active":true,"linkedCount":0,"scanEnabled":false,"appWorkflowId":3,"preBuildStage":{"id":5,"type":"PRE_CI","steps":[{"id":9,"name":"K6 Load testing","description":"K6 is an open-source tool and cloud service that makes load testing easy for developers and QA engineers.","index":1,"stepType":"REF_PLUGIN","outputDirectoryPath":null,"inlineStepDetail":null,"pluginRefStepDetail":{"pluginId":1,"inputVariables":[{"id":44,"name":"RelativePathToScript","format":"STRING","description":"checkout path + script path along with script name","isExposed":true,"defaultValue":"/./script.js","value":"sfds","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""},{"id":45,"name":"PrometheusUsername","format":"STRING","description":"username of prometheus account","isExposed":true,"allowEmptyValue":true,"value":"sdf","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""},{"id":46,"name":"PrometheusApiKey","format":"STRING","description":"api key of prometheus account","isExposed":true,"allowEmptyValue":true,"value":"hter","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""},{"id":47,"name":"PrometheusRemoteWriteEndpoint","format":"STRING","description":"git write endpoint of prometheus account","isExposed":true,"allowEmptyValue":true,"value":"ewq","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""},{"id":48,"name":"OutputType","format":"STRING","description":"output type - LOG or PROMETHEUS","isExposed":true,"defaultValue":"LOG","value":"erg","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""}],"outputVariables":null,"conditionDetails":null},"triggerIfParentStageFail":false},{"id":10,"name":"Task 3","description":"","index":3,"stepType":"INLINE","outputDirectoryPath":["./asap"],"inlineStepDetail":{"scriptType":"CONTAINER_IMAGE","script":null,"storeScriptAt":null,"mountDirectoryFromHost":false,"containerImagePath":"alpine:latest","commandArgsMap":[{"command":"echo","args":["HOSTNAME"]}],"inputVariables":null,"outputVariables":null,"conditionDetails":null,"portMap":[],"mountCodeToContainerPath":null,"mountPathMap":null},"pluginRefStepDetail":null,"triggerIfParentStageFail":false},{"id":8,"name":"Task 1","description":"","index":2,"stepType":"INLINE","outputDirectoryPath":["./test"],"inlineStepDetail":{"scriptType":"SHELL","script":"#!/bin/sh \nset -eo pipefail \necho \"Hello from inside pre-build stage\"\n#set -v ## uncomment this to debug the script \n","storeScriptAt":"","mountDirectoryFromHost":false,"commandArgsMap":[{"command":"","args":null}],"inputVariables":null,"outputVariables":null,"conditionDetails":null},"pluginRefStepDetail":null,"triggerIfParentStageFail":false}]},"postBuildStage":{},"isDockerConfigOverridden":false,"dockerConfigOverride":{}}}` +const CiPipelineStageDeleteReqJson = `{"appId":1,"appWorkflowId":8,"action":2,"ciPipeline":{"isManual":false,"dockerArgs":{},"isExternal":false,"parentCiPipeline":0,"parentAppId":0,"appId":1,"externalCiConfig":{"id":0,"webhookUrl":"","payload":"","accessKey":"","payloadOption":null,"schema":null,"responses":null,"projectId":0,"projectName":"","environmentId":"","environmentName":"","environmentIdentifier":"","appId":0,"appName":"","role":""},"ciMaterial":[{"gitMaterialId":1,"id":8,"source":{"type":"SOURCE_TYPE_BRANCH_FIXED","value":"main","regex":""}}],"name":"ci-1-unjn","id":8,"active":true,"linkedCount":0,"scanEnabled":false,"appWorkflowId":8,"preBuildStage":{"id":7,"type":"PRE_CI","steps":[{"id":11,"name":"Task 1","description":"","index":1,"stepType":"INLINE","outputDirectoryPath":["./test"],"inlineStepDetail":{"scriptType":"SHELL","script":"#!/bin/sh \nset -eo pipefail \necho \"Prakash\"\n#set -v ## uncomment this to debug the script \n","storeScriptAt":"","mountDirectoryFromHost":false,"commandArgsMap":[{"command":"","args":null}],"inputVariables":null,"outputVariables":null,"conditionDetails":null},"pluginRefStepDetail":null,"triggerIfParentStageFail":false},{"id":12,"name":"K6 Load testing","description":"K6 is an open-source tool and cloud service that makes load testing easy for developers and QA engineers.","index":2,"stepType":"REF_PLUGIN","outputDirectoryPath":null,"inlineStepDetail":null,"pluginRefStepDetail":{"pluginId":1,"inputVariables":[{"id":49,"name":"RelativePathToScript","format":"STRING","description":"checkout path + script path along with script name","isExposed":true,"defaultValue":"/./script.js","value":"sfds","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""},{"id":50,"name":"PrometheusUsername","format":"STRING","description":"username of prometheus account","isExposed":true,"allowEmptyValue":true,"value":"sdf","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""},{"id":51,"name":"PrometheusApiKey","format":"STRING","description":"api key of prometheus account","isExposed":true,"allowEmptyValue":true,"value":"hter","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""},{"id":52,"name":"PrometheusRemoteWriteEndpoint","format":"STRING","description":"git write endpoint of prometheus account","isExposed":true,"allowEmptyValue":true,"value":"ewq","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""},{"id":53,"name":"OutputType","format":"STRING","description":"output type - LOG or PROMETHEUS","isExposed":true,"defaultValue":"LOG","value":"erg","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""}],"outputVariables":null,"conditionDetails":null},"triggerIfParentStageFail":false},{"id":13,"name":"Task 3","description":"","index":3,"stepType":"INLINE","outputDirectoryPath":["./asap"],"inlineStepDetail":{"scriptType":"CONTAINER_IMAGE","script":"","storeScriptAt":"","mountDirectoryFromHost":false,"containerImagePath":"alpine:latest","commandArgsMap":[{"command":"echo","args":["HOSTNAME"]}],"inputVariables":null,"outputVariables":null,"conditionDetails":null,"portMap":[]},"pluginRefStepDetail":null,"triggerIfParentStageFail":false}]},"isDockerConfigOverridden":false,"dockerConfigOverride":{},"postBuildStage":{}}}` const ScopedVariablePayload = `{"variables":[{"definition":{"varName":"KAFKA","dataType":"primitive","varType":"public","description":"Enter any notes for additional details","shortDescription":"Some description for variables"},"attributeValue":[{"variableValue":{"value":"Global"},"attributeType":"Global","attributeParams":null}]},{"definition":{"varName":"NewVariable","dataType":"primitive","varType":"private","description":"New Notes","shortDescription":"New Descriotion"},"attributeValue":[{"variableValue":{"value":"gg"},"attributeType":"Global","attributeParams":null}]},{"definition":{"varName":"test-var-1","dataType":"primitive","varType":"public","description":"","shortDescription":""},"attributeValue":[{"variableValue":{"value":"test1"},"attributeType":"Global","attributeParams":null}]},{"definition":{"varName":"test-var-2","dataType":"primitive","varType":"public","description":"","shortDescription":""},"attributeValue":[{"variableValue":{"value":"test2"},"attributeType":"Global","attributeParams":null}]}]}` var ciPipelineId int @@ -143,7 +143,7 @@ var pipelineStageReq = &bean.PipelineStageDto{ Id: 4, Name: "PrometheusRemoteWriteEndpoint", Format: "STRING", - Description: "remote write endpoint of prometheus account", + Description: "git write endpoint of prometheus account", IsExposed: true, AllowEmptyValue: true, DefaultValue: "", @@ -323,7 +323,7 @@ var pipelineStageReqUnresolved = &bean.PipelineStageDto{ Id: 4, Name: "PrometheusRemoteWriteEndpoint", Format: "STRING", - Description: "remote write endpoint of prometheus account", + Description: "git write endpoint of prometheus account", IsExposed: true, AllowEmptyValue: true, DefaultValue: "", @@ -504,7 +504,7 @@ var pipelineStageReqResolved = &bean.PipelineStageDto{ Id: 4, Name: "PrometheusRemoteWriteEndpoint", Format: "STRING", - Description: "remote write endpoint of prometheus account", + Description: "git write endpoint of prometheus account", IsExposed: true, AllowEmptyValue: true, DefaultValue: "", diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index a5c8783ce1..9b1aead59a 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -23,7 +23,7 @@ import ( errors3 "errors" "fmt" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" - "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" bean5 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/bean" @@ -193,7 +193,7 @@ type WorkflowDagExecutorImpl struct { deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService chartRefService chartRef.ChartRefService gitOpsConfigReadService config.GitOpsConfigReadService - gitOpsRemoteOperationService remote.GitOpsRemoteOperationService + gitOpsRemoteOperationService git.GitOpsRemoteOperationService } const kedaAutoscaling = "kedaAutoscaling" @@ -268,7 +268,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi user user.UserService, groupRepository repository.DeploymentGroupRepository, envRepository repository2.EnvironmentRepository, - enforcerUtil rbac.EnforcerUtil, eventFactory client.EventFactory, + enforcerUtil rbac.EnforcerUtil, eventFactory client.EventFactory, eventClient client.EventClient, cvePolicyRepository security.CvePolicyRepository, scanResultRepository security.ImageScanResultRepository, appWorkflowRepository appWorkflow.AppWorkflowRepository, @@ -314,7 +314,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService, chartRefService chartRef.ChartRefService, gitOpsConfigReadService config.GitOpsConfigReadService, - gitOpsRemoteOperationService remote.GitOpsRemoteOperationService) *WorkflowDagExecutorImpl { + gitOpsRemoteOperationService git.GitOpsRemoteOperationService) *WorkflowDagExecutorImpl { wde := &WorkflowDagExecutorImpl{logger: Logger, pipelineRepository: pipelineRepository, cdWorkflowRepository: cdWorkflowRepository, diff --git a/scripts/devtron-reference-helm-charts/deployment-chart_1-0-0/app-values.yaml b/scripts/devtron-reference-helm-charts/deployment-chart_1-0-0/app-values.yaml index 0dce06c558..a5bebb5c56 100644 --- a/scripts/devtron-reference-helm-charts/deployment-chart_1-0-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/deployment-chart_1-0-0/app-values.yaml @@ -308,5 +308,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.remote" -# - "bar.remote" +# - "foo.git" +# - "bar.git" diff --git a/scripts/devtron-reference-helm-charts/deployment-chart_1-1-0/app-values.yaml b/scripts/devtron-reference-helm-charts/deployment-chart_1-1-0/app-values.yaml index bbebc7acab..664b444031 100644 --- a/scripts/devtron-reference-helm-charts/deployment-chart_1-1-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/deployment-chart_1-1-0/app-values.yaml @@ -449,5 +449,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.remote" -# - "bar.remote" +# - "foo.git" +# - "bar.git" diff --git a/scripts/devtron-reference-helm-charts/deployment-chart_4-18-0/app-values.yaml b/scripts/devtron-reference-helm-charts/deployment-chart_4-18-0/app-values.yaml index ff562cb579..96bfcafec7 100644 --- a/scripts/devtron-reference-helm-charts/deployment-chart_4-18-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/deployment-chart_4-18-0/app-values.yaml @@ -508,5 +508,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.remote" -# - "bar.remote" +# - "foo.git" +# - "bar.git" diff --git a/scripts/devtron-reference-helm-charts/deployment-chart_4-19-0/app-values.yaml b/scripts/devtron-reference-helm-charts/deployment-chart_4-19-0/app-values.yaml index ce73c2af57..88daa46112 100644 --- a/scripts/devtron-reference-helm-charts/deployment-chart_4-19-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/deployment-chart_4-19-0/app-values.yaml @@ -514,5 +514,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.remote" -# - "bar.remote" +# - "foo.git" +# - "bar.git" diff --git a/scripts/devtron-reference-helm-charts/reference-chart_3-12-0/app-values.yaml b/scripts/devtron-reference-helm-charts/reference-chart_3-12-0/app-values.yaml index 85f565f40a..f524a92391 100644 --- a/scripts/devtron-reference-helm-charts/reference-chart_3-12-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/reference-chart_3-12-0/app-values.yaml @@ -233,5 +233,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.remote" -# - "bar.remote" +# - "foo.git" +# - "bar.git" diff --git a/scripts/devtron-reference-helm-charts/reference-chart_3-13-0/app-values.yaml b/scripts/devtron-reference-helm-charts/reference-chart_3-13-0/app-values.yaml index 85f565f40a..f524a92391 100644 --- a/scripts/devtron-reference-helm-charts/reference-chart_3-13-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/reference-chart_3-13-0/app-values.yaml @@ -233,5 +233,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.remote" -# - "bar.remote" +# - "foo.git" +# - "bar.git" diff --git a/scripts/devtron-reference-helm-charts/reference-chart_4-11-0/app-values.yaml b/scripts/devtron-reference-helm-charts/reference-chart_4-11-0/app-values.yaml index a9eca0e03a..2771ba40ed 100644 --- a/scripts/devtron-reference-helm-charts/reference-chart_4-11-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/reference-chart_4-11-0/app-values.yaml @@ -276,5 +276,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.remote" -# - "bar.remote" +# - "foo.git" +# - "bar.git" diff --git a/scripts/devtron-reference-helm-charts/reference-chart_4-12-0/app-values.yaml b/scripts/devtron-reference-helm-charts/reference-chart_4-12-0/app-values.yaml index a3022d7bb8..e9fdbd4d14 100644 --- a/scripts/devtron-reference-helm-charts/reference-chart_4-12-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/reference-chart_4-12-0/app-values.yaml @@ -289,5 +289,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.remote" -# - "bar.remote" +# - "foo.git" +# - "bar.git" diff --git a/scripts/devtron-reference-helm-charts/reference-chart_4-13-0/app-values.yaml b/scripts/devtron-reference-helm-charts/reference-chart_4-13-0/app-values.yaml index a3022d7bb8..e9fdbd4d14 100644 --- a/scripts/devtron-reference-helm-charts/reference-chart_4-13-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/reference-chart_4-13-0/app-values.yaml @@ -289,5 +289,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.remote" -# - "bar.remote" +# - "foo.git" +# - "bar.git" diff --git a/scripts/devtron-reference-helm-charts/reference-chart_4-14-0/app-values.yaml b/scripts/devtron-reference-helm-charts/reference-chart_4-14-0/app-values.yaml index d73eee8c80..87327b1d51 100644 --- a/scripts/devtron-reference-helm-charts/reference-chart_4-14-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/reference-chart_4-14-0/app-values.yaml @@ -293,5 +293,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.remote" -# - "bar.remote" +# - "foo.git" +# - "bar.git" diff --git a/scripts/devtron-reference-helm-charts/reference-chart_4-15-0/app-values.yaml b/scripts/devtron-reference-helm-charts/reference-chart_4-15-0/app-values.yaml index ff44a3679a..92eb571112 100644 --- a/scripts/devtron-reference-helm-charts/reference-chart_4-15-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/reference-chart_4-15-0/app-values.yaml @@ -306,5 +306,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.remote" -# - "bar.remote" +# - "foo.git" +# - "bar.git" diff --git a/scripts/devtron-reference-helm-charts/reference-chart_4-16-0/app-values.yaml b/scripts/devtron-reference-helm-charts/reference-chart_4-16-0/app-values.yaml index fe18509efd..a93d0fb412 100644 --- a/scripts/devtron-reference-helm-charts/reference-chart_4-16-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/reference-chart_4-16-0/app-values.yaml @@ -308,5 +308,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.remote" -# - "bar.remote" +# - "foo.git" +# - "bar.git" diff --git a/scripts/devtron-reference-helm-charts/reference-chart_4-17-0/app-values.yaml b/scripts/devtron-reference-helm-charts/reference-chart_4-17-0/app-values.yaml index a621f49a79..80ccc7cb32 100644 --- a/scripts/devtron-reference-helm-charts/reference-chart_4-17-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/reference-chart_4-17-0/app-values.yaml @@ -360,5 +360,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.remote" -# - "bar.remote" +# - "foo.git" +# - "bar.git" diff --git a/scripts/devtron-reference-helm-charts/reference-chart_4-18-0/app-values.yaml b/scripts/devtron-reference-helm-charts/reference-chart_4-18-0/app-values.yaml index f4c8cef663..8079c1e887 100644 --- a/scripts/devtron-reference-helm-charts/reference-chart_4-18-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/reference-chart_4-18-0/app-values.yaml @@ -424,5 +424,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.remote" -# - "bar.remote" +# - "foo.git" +# - "bar.git" diff --git a/scripts/devtron-reference-helm-charts/reference-chart_5-0-0/app-values.yaml b/scripts/devtron-reference-helm-charts/reference-chart_5-0-0/app-values.yaml index 0555153aee..fd93c244d8 100644 --- a/scripts/devtron-reference-helm-charts/reference-chart_5-0-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/reference-chart_5-0-0/app-values.yaml @@ -423,5 +423,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.remote" -# - "bar.remote" +# - "foo.git" +# - "bar.git" diff --git a/scripts/devtron-reference-helm-charts/statefulset-chart_4-18-0/app-values.yaml b/scripts/devtron-reference-helm-charts/statefulset-chart_4-18-0/app-values.yaml index f1d80a0ea3..1a8063c237 100644 --- a/scripts/devtron-reference-helm-charts/statefulset-chart_4-18-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/statefulset-chart_4-18-0/app-values.yaml @@ -393,5 +393,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.remote" -# - "bar.remote" +# - "foo.git" +# - "bar.git" diff --git a/scripts/devtron-reference-helm-charts/statefulset-chart_4-19-0/app-values.yaml b/scripts/devtron-reference-helm-charts/statefulset-chart_4-19-0/app-values.yaml index ad27515a44..f7edfb417e 100644 --- a/scripts/devtron-reference-helm-charts/statefulset-chart_4-19-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/statefulset-chart_4-19-0/app-values.yaml @@ -397,5 +397,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.remote" -# - "bar.remote" +# - "foo.git" +# - "bar.git" diff --git a/scripts/devtron-reference-helm-charts/statefulset-chart_5-0-0/app-values.yaml b/scripts/devtron-reference-helm-charts/statefulset-chart_5-0-0/app-values.yaml index 1b1912aa68..4a057c104f 100644 --- a/scripts/devtron-reference-helm-charts/statefulset-chart_5-0-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/statefulset-chart_5-0-0/app-values.yaml @@ -384,5 +384,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.remote" -# - "bar.remote" +# - "foo.git" +# - "bar.git" diff --git a/scripts/sql/49_plugin_integration.up.sql b/scripts/sql/49_plugin_integration.up.sql index e52a73ccbe..2fd34bc5b9 100644 --- a/scripts/sql/49_plugin_integration.up.sql +++ b/scripts/sql/49_plugin_integration.up.sql @@ -307,11 +307,11 @@ then export GOCACHE=/usr/local/go/cache export PATH=$PATH:/usr/local/go/bin go install go.k6.io/xk6/cmd/xk6@latest - xk6 build --with github.com/grafana/xk6-output-prometheus-remote + xk6 build --with github.com/grafana/xk6-output-prometheus-git K6_PROMETHEUS_USER=$PrometheusUsername \ K6_PROMETHEUS_PASSWORD=$PrometheusApiKey \ K6_PROMETHEUS_REMOTE_URL=$PrometheusRemoteWriteEndpoint \ - ./k6 run $PathToScript -o output-prometheus-remote + ./k6 run $PathToScript -o output-prometheus-git elif [ $OutputType == "LOG" ] then docker pull grafana/k6 @@ -343,7 +343,7 @@ INSERT INTO "public"."plugin_step_variable" ("id", "plugin_step_id", "name", "fo ('1', '1','RelativePathToScript','STRING','checkout path + script path along with script name','t','f','INPUT','NEW','/./script.js','1','f','now()', '1', 'now()', '1'), ('2', '1','PrometheusUsername','STRING','username of prometheus account','t','t','INPUT','NEW',null, '1' ,'f','now()', '1', 'now()', '1'), ('3', '1','PrometheusApiKey','STRING','api key of prometheus account','t','t','INPUT','NEW',null, '1','f','now()', '1', 'now()', '1'), -('4', '1','PrometheusRemoteWriteEndpoint','STRING','remote write endpoint of prometheus account','t','t','INPUT','NEW',null, '1','f','now()', '1', 'now()', '1'), +('4', '1','PrometheusRemoteWriteEndpoint','STRING','git write endpoint of prometheus account','t','t','INPUT','NEW',null, '1','f','now()', '1', 'now()', '1'), ('5', '1','OutputType','STRING','output type - LOG or PROMETHEUS','t','f','INPUT','NEW','LOG', '1','f','now()', '1', 'now()', '1'), ('6', '2','SonarqubeProjectKey','STRING','project key of grafana sonarqube account','t','t','INPUT','NEW',null, '1', 'f','now()', '1', 'now()', '1'), ('7', '2','SonarqubeApiKey','STRING','api key of sonarqube account','t','t','INPUT','NEW',null, '1', 'f','now()', '1', 'now()', '1'), diff --git a/util/RequestUtil.go b/util/RequestUtil.go index 3eb349bc3d..eb0b251ec9 100644 --- a/util/RequestUtil.go +++ b/util/RequestUtil.go @@ -24,7 +24,7 @@ import ( const xForwardedForHeaderName = "X-Forwarded-For" // GetClientIP gets a requests IP address by reading off the forwarded-for -// header (for proxies) and falls back to use the remote address. +// header (for proxies) and falls back to use the git address. func GetClientIP(r *http.Request) string { xForwardedFor := r.Header.Get(xForwardedForHeaderName) if len(xForwardedFor) > 0 { diff --git a/wire_gen.go b/wire_gen.go index 00bc3d66ab..bcea4e49e5 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -105,7 +105,7 @@ import ( "github.com/devtron-labs/devtron/pkg/commonService" delete2 "github.com/devtron-labs/devtron/pkg/delete" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" - "github.com/devtron-labs/devtron/pkg/deployment/gitOps/remote" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" repository11 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/repository" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate" @@ -118,7 +118,7 @@ import ( "github.com/devtron-labs/devtron/pkg/generateManifest" "github.com/devtron-labs/devtron/pkg/genericNotes" repository12 "github.com/devtron-labs/devtron/pkg/genericNotes/repository" - "github.com/devtron-labs/devtron/pkg/git" + git2 "github.com/devtron-labs/devtron/pkg/git" "github.com/devtron-labs/devtron/pkg/gitops" k8s2 "github.com/devtron-labs/devtron/pkg/k8s" application2 "github.com/devtron-labs/devtron/pkg/k8s/application" @@ -377,7 +377,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - gitOpsRemoteOperationServiceImpl := remote.NewGitOpsRemoteOperationServiceImpl(sugaredLogger, gitFactory, gitOpsConfigReadServiceImpl, chartTemplateServiceImpl) + gitOpsRemoteOperationServiceImpl := git.NewGitOpsRemoteOperationServiceImpl(sugaredLogger, gitFactory, gitOpsConfigReadServiceImpl, chartTemplateServiceImpl) appServiceImpl := app2.NewAppService(envConfigOverrideRepositoryImpl, pipelineOverrideRepositoryImpl, mergeUtil, sugaredLogger, pipelineRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, applicationServiceClientImpl, appRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, chartTemplateServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, appStatusServiceImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, globalEnvVariables, scopedVariableCMCSManagerImpl, acdConfig, chartRefServiceImpl, gitOpsConfigReadServiceImpl, gitOpsRemoteOperationServiceImpl) validate, err := util.IntValidator() if err != nil { @@ -574,7 +574,7 @@ func InitializeApp() (*App, error) { clusterRestHandlerImpl := cluster3.NewClusterRestHandlerImpl(clusterServiceImplExtended, genericNoteServiceImpl, clusterDescriptionServiceImpl, sugaredLogger, userServiceImpl, validate, enforcerImpl, deleteServiceExtendedImpl, argoUserServiceImpl, environmentServiceImpl, clusterRbacServiceImpl) clusterRouterImpl := cluster3.NewClusterRouterImpl(clusterRestHandlerImpl) gitWebhookRepositoryImpl := repository.NewGitWebhookRepositoryImpl(db) - gitWebhookServiceImpl := git.NewGitWebhookServiceImpl(sugaredLogger, ciHandlerImpl, gitWebhookRepositoryImpl) + gitWebhookServiceImpl := git2.NewGitWebhookServiceImpl(sugaredLogger, ciHandlerImpl, gitWebhookRepositoryImpl) gitWebhookRestHandlerImpl := restHandler.NewGitWebhookRestHandlerImpl(sugaredLogger, gitWebhookServiceImpl) webhookServiceImpl := pipeline.NewWebhookServiceImpl(ciArtifactRepositoryImpl, sugaredLogger, ciPipelineRepositoryImpl, appServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciWorkflowRepositoryImpl, workflowDagExecutorImpl, ciHandlerImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, customTagServiceImpl) ciEventConfig, err := pubsub.GetCiEventConfig() @@ -729,7 +729,7 @@ func InitializeApp() (*App, error) { } bulkUpdateRestHandlerImpl := restHandler.NewBulkUpdateRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, bulkUpdateServiceImpl, chartServiceImpl, propertiesConfigServiceImpl, applicationServiceClientImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, environmentServiceImpl, gitRegistryConfigImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, appWorkflowServiceImpl, materialRepositoryImpl, policyServiceImpl, imageScanResultRepositoryImpl, argoUserServiceImpl) bulkUpdateRouterImpl := router.NewBulkUpdateRouterImpl(bulkUpdateRestHandlerImpl) - webhookSecretValidatorImpl := git.NewWebhookSecretValidatorImpl(sugaredLogger) + webhookSecretValidatorImpl := git2.NewWebhookSecretValidatorImpl(sugaredLogger) webhookEventHandlerImpl := restHandler.NewWebhookEventHandlerImpl(sugaredLogger, gitHostConfigImpl, eventRESTClientImpl, webhookSecretValidatorImpl, webhookEventDataConfigImpl) webhookListenerRouterImpl := router.NewWebhookListenerRouterImpl(webhookEventHandlerImpl) appRestHandlerImpl := restHandler.NewAppRestHandlerImpl(sugaredLogger, appCrudOperationServiceImpl, userServiceImpl, validate, enforcerUtilImpl, enforcerImpl, helmAppServiceImpl, enforcerUtilHelmImpl, genericNoteServiceImpl) From fcf03ac6f920105a1372d86486917e34bc2c79b1 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 23 Jan 2024 12:21:26 +0530 Subject: [PATCH 32/64] renamed gitOpsRemoteOpService If and impl --- cmd/external-app/wire_gen.go | 8 ++--- pkg/app/AppService.go | 4 +-- pkg/app/ManifestPushService.go | 4 +-- .../common/AppStoreDeploymentCommonService.go | 4 +-- .../AppStoreDeploymentFullModeService.go | 4 +-- .../service/AppStoreDeploymentService.go | 4 +-- .../deployment/service/InstalledAppService.go | 4 +-- .../tool/AppStoreDeploymentArgoCdService.go | 4 +-- .../tool/AppStoreDeploymentHelmService.go | 4 +-- ...ationService.go => GitOperationService.go} | 29 ++++++++++--------- pkg/deployment/gitOps/wire_gitOps.go | 4 +-- .../DeploymentPipelineConfigService.go | 4 +-- pkg/pipeline/WorkflowDagExecutor.go | 4 +-- wire_gen.go | 22 +++++++------- 14 files changed, 52 insertions(+), 51 deletions(-) rename pkg/deployment/gitOps/git/{GitOpsRemoteOperationService.go => GitOperationService.go} (90%) diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index b41b6fff67..1f768a004f 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -244,8 +244,8 @@ func InitializeApp() (*App, error) { return nil, err } gitOpsConfigReadServiceImpl := config.NewGitOpsConfigReadServiceImpl(sugaredLogger, gitOpsConfigRepositoryImpl, userServiceImpl, globalEnvVariables) - gitOpsRemoteOperationServiceImpl := git.NewGitOpsRemoteOperationServiceImpl(sugaredLogger, gitFactory, gitOpsConfigReadServiceImpl, chartTemplateServiceImpl) - appStoreDeploymentCommonServiceImpl := appStoreDeploymentCommon.NewAppStoreDeploymentCommonServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, chartTemplateServiceImpl, gitFactory, gitOpsConfigReadServiceImpl, gitOpsRemoteOperationServiceImpl) + gitOperationServiceImpl := git.NewGitOperationServiceImpl(sugaredLogger, gitFactory, gitOpsConfigReadServiceImpl, chartTemplateServiceImpl) + appStoreDeploymentCommonServiceImpl := appStoreDeploymentCommon.NewAppStoreDeploymentCommonServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, chartTemplateServiceImpl, gitFactory, gitOpsConfigReadServiceImpl, gitOperationServiceImpl) attributesServiceImpl := attributes.NewAttributesServiceImpl(sugaredLogger, attributesRepositoryImpl) helmAppRestHandlerImpl := client2.NewHelmAppRestHandlerImpl(sugaredLogger, helmAppServiceImpl, enforcerImpl, clusterServiceImpl, enforcerUtilHelmImpl, appStoreDeploymentCommonServiceImpl, userServiceImpl, attributesServiceImpl, serverEnvConfigServerEnvConfig) helmAppRouterImpl := client2.NewHelmAppRouterImpl(helmAppRestHandlerImpl) @@ -276,7 +276,7 @@ func InitializeApp() (*App, error) { appStoreValuesRouterImpl := appStoreValues.NewAppStoreValuesRouterImpl(appStoreValuesRestHandlerImpl) chartGroupDeploymentRepositoryImpl := repository8.NewChartGroupDeploymentRepositoryImpl(db, sugaredLogger) clusterInstalledAppsRepositoryImpl := repository4.NewClusterInstalledAppsRepositoryImpl(db, sugaredLogger) - appStoreDeploymentHelmServiceImpl := appStoreDeploymentTool.NewAppStoreDeploymentHelmServiceImpl(sugaredLogger, helmAppServiceImpl, appStoreApplicationVersionRepositoryImpl, helmAppClientImpl, installedAppRepositoryImpl, appStoreDeploymentCommonServiceImpl, ociRegistryConfigRepositoryImpl, gitOpsRemoteOperationServiceImpl) + appStoreDeploymentHelmServiceImpl := appStoreDeploymentTool.NewAppStoreDeploymentHelmServiceImpl(sugaredLogger, helmAppServiceImpl, appStoreApplicationVersionRepositoryImpl, helmAppClientImpl, installedAppRepositoryImpl, appStoreDeploymentCommonServiceImpl, ociRegistryConfigRepositoryImpl, gitOperationServiceImpl) installedAppVersionHistoryRepositoryImpl := repository4.NewInstalledAppVersionHistoryRepositoryImpl(sugaredLogger, db) deploymentServiceTypeConfig, err := service3.GetDeploymentServiceTypeConfig() if err != nil { @@ -286,7 +286,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - appStoreDeploymentServiceImpl := service3.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, clusterInstalledAppsRepositoryImpl, appRepositoryImpl, appStoreDeploymentHelmServiceImpl, appStoreDeploymentHelmServiceImpl, environmentServiceImpl, clusterServiceImpl, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, installedAppVersionHistoryRepositoryImpl, gitOpsConfigRepositoryImpl, deploymentServiceTypeConfig, acdConfig, gitOpsConfigReadServiceImpl, gitOpsRemoteOperationServiceImpl) + appStoreDeploymentServiceImpl := service3.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, clusterInstalledAppsRepositoryImpl, appRepositoryImpl, appStoreDeploymentHelmServiceImpl, appStoreDeploymentHelmServiceImpl, environmentServiceImpl, clusterServiceImpl, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, installedAppVersionHistoryRepositoryImpl, gitOpsConfigRepositoryImpl, deploymentServiceTypeConfig, acdConfig, gitOpsConfigReadServiceImpl, gitOperationServiceImpl) appStoreDeploymentRestHandlerImpl := appStoreDeployment.NewAppStoreDeploymentRestHandlerImpl(sugaredLogger, userServiceImpl, enforcerImpl, enforcerUtilImpl, enforcerUtilHelmImpl, appStoreDeploymentServiceImpl, validate, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, helmUserServiceImpl, attributesServiceImpl) appStoreDeploymentRouterImpl := appStoreDeployment.NewAppStoreDeploymentRouterImpl(appStoreDeploymentRestHandlerImpl) chartProviderServiceImpl := chartProvider.NewChartProviderServiceImpl(sugaredLogger, chartRepoRepositoryImpl, chartRepositoryServiceImpl, dockerArtifactStoreRepositoryImpl, ociRegistryConfigRepositoryImpl) diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index 173f503083..34c10331e1 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -122,7 +122,7 @@ type AppServiceImpl struct { acdConfig *argocdServer.ACDConfig chartRefService chartRef.ChartRefService gitOpsConfigReadService config.GitOpsConfigReadService - gitOpsRemoteOperationService git.GitOpsRemoteOperationService + gitOpsRemoteOperationService git.GitOperationService } type AppService interface { @@ -177,7 +177,7 @@ func NewAppService( scopedVariableManager variables.ScopedVariableCMCSManager, acdConfig *argocdServer.ACDConfig, chartRefService chartRef.ChartRefService, gitOpsConfigReadService config.GitOpsConfigReadService, - gitOpsRemoteOperationService git.GitOpsRemoteOperationService) *AppServiceImpl { + gitOpsRemoteOperationService git.GitOperationService) *AppServiceImpl { appServiceImpl := &AppServiceImpl{ environmentConfigRepository: environmentConfigRepository, mergeUtil: mergeUtil, diff --git a/pkg/app/ManifestPushService.go b/pkg/app/ManifestPushService.go index 3e407dd935..a3cc646729 100644 --- a/pkg/app/ManifestPushService.go +++ b/pkg/app/ManifestPushService.go @@ -31,7 +31,7 @@ type GitOpsManifestPushServiceImpl struct { acdConfig *argocdServer.ACDConfig chartRefService chartRef.ChartRefService gitOpsConfigReadService config.GitOpsConfigReadService - gitOpsRemoteOperationService git.GitOpsRemoteOperationService + gitOpsRemoteOperationService git.GitOperationService } func NewGitOpsManifestPushServiceImpl(logger *zap.SugaredLogger, @@ -39,7 +39,7 @@ func NewGitOpsManifestPushServiceImpl(logger *zap.SugaredLogger, pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository, acdConfig *argocdServer.ACDConfig, chartRefService chartRef.ChartRefService, gitOpsConfigReadService config.GitOpsConfigReadService, - gitOpsRemoteOperationService git.GitOpsRemoteOperationService) *GitOpsManifestPushServiceImpl { + gitOpsRemoteOperationService git.GitOperationService) *GitOpsManifestPushServiceImpl { return &GitOpsManifestPushServiceImpl{ logger: logger, pipelineStatusTimelineService: pipelineStatusTimelineService, diff --git a/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go b/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go index 7cbcb99d44..759ed4bd7a 100644 --- a/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go +++ b/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go @@ -69,7 +69,7 @@ type AppStoreDeploymentCommonServiceImpl struct { chartTemplateService util.ChartTemplateService gitFactory *util.GitFactory gitOpsConfigReadService config.GitOpsConfigReadService - gitOpsRemoteOperationService git.GitOpsRemoteOperationService + gitOpsRemoteOperationService git.GitOperationService } func NewAppStoreDeploymentCommonServiceImpl( @@ -80,7 +80,7 @@ func NewAppStoreDeploymentCommonServiceImpl( chartTemplateService util.ChartTemplateService, gitFactory *util.GitFactory, gitOpsConfigReadService config.GitOpsConfigReadService, - gitOpsRemoteOperationService git.GitOpsRemoteOperationService) *AppStoreDeploymentCommonServiceImpl { + gitOpsRemoteOperationService git.GitOperationService) *AppStoreDeploymentCommonServiceImpl { return &AppStoreDeploymentCommonServiceImpl{ logger: logger, installedAppRepository: installedAppRepository, diff --git a/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go b/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go index 42f241b82a..e0a3e066f3 100644 --- a/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go +++ b/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go @@ -73,7 +73,7 @@ type AppStoreDeploymentFullModeServiceImpl struct { installedAppRepositoryHistory repository4.InstalledAppVersionHistoryRepository ACDConfig *argocdServer.ACDConfig gitOpsConfigReadService config.GitOpsConfigReadService - gitOpsRemoteOperationService git.GitOpsRemoteOperationService + gitOpsRemoteOperationService git.GitOperationService } func NewAppStoreDeploymentFullModeServiceImpl(logger *zap.SugaredLogger, @@ -86,7 +86,7 @@ func NewAppStoreDeploymentFullModeServiceImpl(logger *zap.SugaredLogger, installedAppRepositoryHistory repository4.InstalledAppVersionHistoryRepository, ACDConfig *argocdServer.ACDConfig, gitOpsConfigReadService config.GitOpsConfigReadService, - gitOpsRemoteOperationService git.GitOpsRemoteOperationService) *AppStoreDeploymentFullModeServiceImpl { + gitOpsRemoteOperationService git.GitOperationService) *AppStoreDeploymentFullModeServiceImpl { appStoreDeploymentFullModeServiceImpl := &AppStoreDeploymentFullModeServiceImpl{ logger: logger, acdClient: acdClient, diff --git a/pkg/appStore/deployment/service/AppStoreDeploymentService.go b/pkg/appStore/deployment/service/AppStoreDeploymentService.go index cc40832ae0..39b2d05519 100644 --- a/pkg/appStore/deployment/service/AppStoreDeploymentService.go +++ b/pkg/appStore/deployment/service/AppStoreDeploymentService.go @@ -108,7 +108,7 @@ type AppStoreDeploymentServiceImpl struct { deploymentTypeConfig *DeploymentServiceTypeConfig aCDConfig *argocdServer.ACDConfig gitOpsConfigReadService config.GitOpsConfigReadService - gitOpsRemoteOperationService git.GitOpsRemoteOperationService + gitOpsRemoteOperationService git.GitOperationService } func NewAppStoreDeploymentServiceImpl(logger *zap.SugaredLogger, installedAppRepository repository.InstalledAppRepository, @@ -120,7 +120,7 @@ func NewAppStoreDeploymentServiceImpl(logger *zap.SugaredLogger, installedAppRep installedAppRepositoryHistory repository.InstalledAppVersionHistoryRepository, gitOpsRepository repository2.GitOpsConfigRepository, deploymentTypeConfig *DeploymentServiceTypeConfig, aCDConfig *argocdServer.ACDConfig, gitOpsConfigReadService config.GitOpsConfigReadService, - gitOpsRemoteOperationService git.GitOpsRemoteOperationService) *AppStoreDeploymentServiceImpl { + gitOpsRemoteOperationService git.GitOperationService) *AppStoreDeploymentServiceImpl { appStoreDeploymentServiceImpl := &AppStoreDeploymentServiceImpl{ logger: logger, diff --git a/pkg/appStore/deployment/service/InstalledAppService.go b/pkg/appStore/deployment/service/InstalledAppService.go index 4f33452280..0bac817246 100644 --- a/pkg/appStore/deployment/service/InstalledAppService.go +++ b/pkg/appStore/deployment/service/InstalledAppService.go @@ -124,7 +124,7 @@ type InstalledAppServiceImpl struct { k8sApplicationService application3.K8sApplicationService acdConfig *argocdServer.ACDConfig gitOpsConfigReadService config.GitOpsConfigReadService - gitOpsRemoteOperationService git.GitOpsRemoteOperationService + gitOpsRemoteOperationService git.GitOperationService } func NewInstalledAppServiceImpl(logger *zap.SugaredLogger, @@ -147,7 +147,7 @@ func NewInstalledAppServiceImpl(logger *zap.SugaredLogger, appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService, k8sCommonService k8s.K8sCommonService, k8sApplicationService application3.K8sApplicationService, acdConfig *argocdServer.ACDConfig, gitOpsConfigReadService config.GitOpsConfigReadService, - gitOpsRemoteOperationService git.GitOpsRemoteOperationService) (*InstalledAppServiceImpl, error) { + gitOpsRemoteOperationService git.GitOperationService) (*InstalledAppServiceImpl, error) { impl := &InstalledAppServiceImpl{ logger: logger, installedAppRepository: installedAppRepository, diff --git a/pkg/appStore/deployment/tool/AppStoreDeploymentArgoCdService.go b/pkg/appStore/deployment/tool/AppStoreDeploymentArgoCdService.go index f5973894d5..a19fa6c975 100644 --- a/pkg/appStore/deployment/tool/AppStoreDeploymentArgoCdService.go +++ b/pkg/appStore/deployment/tool/AppStoreDeploymentArgoCdService.go @@ -71,7 +71,7 @@ type AppStoreDeploymentArgoCdServiceImpl struct { appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository argoClientWrapperService argocdServer.ArgoClientWrapperService acdConfig *argocdServer.ACDConfig - gitOpsRemoteOperationService git.GitOpsRemoteOperationService + gitOpsRemoteOperationService git.GitOperationService } func NewAppStoreDeploymentArgoCdServiceImpl(logger *zap.SugaredLogger, appStoreDeploymentFullModeService appStoreDeploymentFullMode.AppStoreDeploymentFullModeService, @@ -83,7 +83,7 @@ func NewAppStoreDeploymentArgoCdServiceImpl(logger *zap.SugaredLogger, appStoreD pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository, appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository, argoClientWrapperService argocdServer.ArgoClientWrapperService, acdConfig *argocdServer.ACDConfig, - gitOpsRemoteOperationService git.GitOpsRemoteOperationService) *AppStoreDeploymentArgoCdServiceImpl { + gitOpsRemoteOperationService git.GitOperationService) *AppStoreDeploymentArgoCdServiceImpl { return &AppStoreDeploymentArgoCdServiceImpl{ Logger: logger, appStoreDeploymentFullModeService: appStoreDeploymentFullModeService, diff --git a/pkg/appStore/deployment/tool/AppStoreDeploymentHelmService.go b/pkg/appStore/deployment/tool/AppStoreDeploymentHelmService.go index 9b42f2bd9f..d8dbfc167a 100644 --- a/pkg/appStore/deployment/tool/AppStoreDeploymentHelmService.go +++ b/pkg/appStore/deployment/tool/AppStoreDeploymentHelmService.go @@ -39,7 +39,7 @@ type AppStoreDeploymentHelmServiceImpl struct { installedAppRepository repository.InstalledAppRepository appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService OCIRegistryConfigRepository repository2.OCIRegistryConfigRepository - gitOpsRemoteOperationService git.GitOpsRemoteOperationService + gitOpsRemoteOperationService git.GitOperationService } func NewAppStoreDeploymentHelmServiceImpl(logger *zap.SugaredLogger, helmAppService client.HelmAppService, @@ -47,7 +47,7 @@ func NewAppStoreDeploymentHelmServiceImpl(logger *zap.SugaredLogger, helmAppServ helmAppClient client.HelmAppClient, installedAppRepository repository.InstalledAppRepository, appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService, OCIRegistryConfigRepository repository2.OCIRegistryConfigRepository, - gitOpsRemoteOperationService git.GitOpsRemoteOperationService) *AppStoreDeploymentHelmServiceImpl { + gitOpsRemoteOperationService git.GitOperationService) *AppStoreDeploymentHelmServiceImpl { return &AppStoreDeploymentHelmServiceImpl{ Logger: logger, helmAppService: helmAppService, diff --git a/pkg/deployment/gitOps/git/GitOpsRemoteOperationService.go b/pkg/deployment/gitOps/git/GitOperationService.go similarity index 90% rename from pkg/deployment/gitOps/git/GitOpsRemoteOperationService.go rename to pkg/deployment/gitOps/git/GitOperationService.go index 375031ad60..e434666a51 100644 --- a/pkg/deployment/gitOps/git/GitOpsRemoteOperationService.go +++ b/pkg/deployment/gitOps/git/GitOperationService.go @@ -17,7 +17,7 @@ import ( "time" ) -type GitOpsRemoteOperationService interface { +type GitOperationService interface { CreateGitRepositoryForApp(gitOpsRepoName, baseTemplateName, version string, userId int32) (chartGitAttribute *commonBean.ChartGitAttribute, err error) PushChartToGitRepo(gitOpsRepoName, referenceTemplate, version, @@ -31,25 +31,26 @@ type GitOpsRemoteOperationService interface { GetRepoUrlByRepoName(repoName string) (string, error) } -type GitOpsRemoteOperationServiceImpl struct { +type GitOperationServiceImpl struct { logger *zap.SugaredLogger gitFactory *util.GitFactory gitOpsConfigReadService config.GitOpsConfigReadService chartTemplateService util.ChartTemplateService } -func NewGitOpsRemoteOperationServiceImpl(logger *zap.SugaredLogger, gitFactory *util.GitFactory, +func NewGitOperationServiceImpl(logger *zap.SugaredLogger, gitFactory *util.GitFactory, gitOpsConfigReadService config.GitOpsConfigReadService, - chartTemplateService util.ChartTemplateService) *GitOpsRemoteOperationServiceImpl { - return &GitOpsRemoteOperationServiceImpl{ + chartTemplateService util.ChartTemplateService) *GitOperationServiceImpl { + return &GitOperationServiceImpl{ logger: logger, + gitFactory: gitFactory, gitOpsConfigReadService: gitOpsConfigReadService, chartTemplateService: chartTemplateService, } } -func (impl *GitOpsRemoteOperationServiceImpl) CreateGitRepositoryForApp(gitOpsRepoName, baseTemplateName, +func (impl *GitOperationServiceImpl) CreateGitRepositoryForApp(gitOpsRepoName, baseTemplateName, version string, userId int32) (chartGitAttribute *commonBean.ChartGitAttribute, err error) { //baseTemplateName replace whitespace space := regexp.MustCompile(`\s+`) @@ -80,7 +81,7 @@ func (impl *GitOpsRemoteOperationServiceImpl) CreateGitRepositoryForApp(gitOpsRe return &commonBean.ChartGitAttribute{RepoUrl: repoUrl, ChartLocation: filepath.Join(baseTemplateName, version)}, nil } -func (impl *GitOpsRemoteOperationServiceImpl) PushChartToGitRepo(gitOpsRepoName, referenceTemplate, version, +func (impl *GitOperationServiceImpl) PushChartToGitRepo(gitOpsRepoName, referenceTemplate, version, tempReferenceTemplateDir string, repoUrl string, userId int32) (err error) { chartDir := fmt.Sprintf("%s-%s", gitOpsRepoName, impl.chartTemplateService.GetDir()) clonedDir := impl.gitFactory.GitService.GetCloneDirectory(chartDir) @@ -159,7 +160,7 @@ func (impl *GitOpsRemoteOperationServiceImpl) PushChartToGitRepo(gitOpsRepoName, return nil } -func (impl *GitOpsRemoteOperationServiceImpl) CreateReadmeInGitRepo(gitOpsRepoName string, userId int32) error { +func (impl *GitOperationServiceImpl) CreateReadmeInGitRepo(gitOpsRepoName string, userId int32) error { userEmailId, userName := impl.gitOpsConfigReadService.GetUserEmailIdAndNameForGitOpsCommit(userId) gitOpsConfig, err := impl.gitOpsConfigReadService.GetGitOpsConfigActive() if err != nil { @@ -179,7 +180,7 @@ func (impl *GitOpsRemoteOperationServiceImpl) CreateReadmeInGitRepo(gitOpsRepoNa return nil } -func (impl *GitOpsRemoteOperationServiceImpl) CreateChartProxy(chartMetaData *chart.Metadata, refChartLocation string, envName string, +func (impl *GitOperationServiceImpl) CreateChartProxy(chartMetaData *chart.Metadata, refChartLocation string, envName string, chartProxyReq *bean.ChartProxyReqDto) (string, *commonBean.ChartGitAttribute, error) { chartMetaData.ApiVersion = "v2" // ensure always v2 dir := impl.chartTemplateService.GetDir() @@ -220,7 +221,7 @@ func (impl *GitOpsRemoteOperationServiceImpl) CreateChartProxy(chartMetaData *ch return valuesYaml, chartGitAttr, nil } -func (impl *GitOpsRemoteOperationServiceImpl) createAndPushToGitChartProxy(appStoreName, tmpChartLocation string, envName string, +func (impl *GitOperationServiceImpl) createAndPushToGitChartProxy(appStoreName, tmpChartLocation string, envName string, chartProxyReq *bean.ChartProxyReqDto) (chartGitAttribute *commonBean.ChartGitAttribute, err error) { //baseTemplateName replace whitespace space := regexp.MustCompile(`\s+`) @@ -305,7 +306,7 @@ func (impl *GitOpsRemoteOperationServiceImpl) createAndPushToGitChartProxy(appSt return &commonBean.ChartGitAttribute{RepoUrl: repoUrl, ChartLocation: filepath.Join("", acdAppName)}, nil } -func (impl *GitOpsRemoteOperationServiceImpl) GitPull(clonedDir string, repoUrl string, appStoreName string) error { +func (impl *GitOperationServiceImpl) GitPull(clonedDir string, repoUrl string, appStoreName string) error { err := impl.gitFactory.GitService.Pull(clonedDir) //TODO check for local repo exists before clone if err != nil { impl.logger.Errorw("error in pulling git", "clonedDir", clonedDir, "err", err) @@ -319,7 +320,7 @@ func (impl *GitOpsRemoteOperationServiceImpl) GitPull(clonedDir string, repoUrl return nil } -func (impl *GitOpsRemoteOperationServiceImpl) CommitValues(chartGitAttr *util.ChartConfig) (commitHash string, commitTime time.Time, err error) { +func (impl *GitOperationServiceImpl) CommitValues(chartGitAttr *util.ChartConfig) (commitHash string, commitTime time.Time, err error) { bitbucketMetadata, err := impl.gitOpsConfigReadService.GetBitbucketMetadata() if err != nil { impl.logger.Errorw("error in getting bitbucket metadata", "err", err) @@ -337,7 +338,7 @@ func (impl *GitOpsRemoteOperationServiceImpl) CommitValues(chartGitAttr *util.Ch return commitHash, commitTime, nil } -func (impl *GitOpsRemoteOperationServiceImpl) CreateRepository(dto *bean2.GitOpsConfigDto, userId int32) (string, bool, error) { +func (impl *GitOperationServiceImpl) CreateRepository(dto *bean2.GitOpsConfigDto, userId int32) (string, bool, error) { //getting user name & emailId for commit author data userEmailId, userName := impl.gitOpsConfigReadService.GetUserEmailIdAndNameForGitOpsCommit(userId) if dto != nil { @@ -354,7 +355,7 @@ func (impl *GitOpsRemoteOperationServiceImpl) CreateRepository(dto *bean2.GitOps return repoUrl, isNew, nil } -func (impl *GitOpsRemoteOperationServiceImpl) GetRepoUrlByRepoName(repoName string) (string, error) { +func (impl *GitOperationServiceImpl) GetRepoUrlByRepoName(repoName string) (string, error) { repoUrl := "" bitbucketMetadata, err := impl.gitOpsConfigReadService.GetBitbucketMetadata() if err != nil { diff --git a/pkg/deployment/gitOps/wire_gitOps.go b/pkg/deployment/gitOps/wire_gitOps.go index 92a256a4e4..a0b7daa517 100644 --- a/pkg/deployment/gitOps/wire_gitOps.go +++ b/pkg/deployment/gitOps/wire_gitOps.go @@ -10,6 +10,6 @@ var GitOpsWireSet = wire.NewSet( config.NewGitOpsConfigReadServiceImpl, wire.Bind(new(config.GitOpsConfigReadService), new(*config.GitOpsConfigReadServiceImpl)), - git.NewGitOpsRemoteOperationServiceImpl, - wire.Bind(new(git.GitOpsRemoteOperationService), new(*git.GitOpsRemoteOperationServiceImpl)), + git.NewGitOperationServiceImpl, + wire.Bind(new(git.GitOperationService), new(*git.GitOperationServiceImpl)), ) diff --git a/pkg/pipeline/DeploymentPipelineConfigService.go b/pkg/pipeline/DeploymentPipelineConfigService.go index 273145d716..72a15b312c 100644 --- a/pkg/pipeline/DeploymentPipelineConfigService.go +++ b/pkg/pipeline/DeploymentPipelineConfigService.go @@ -148,7 +148,7 @@ type CdPipelineConfigServiceImpl struct { argoClientWrapperService argocdServer.ArgoClientWrapperService deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService gitOpsConfigReadService config.GitOpsConfigReadService - gitOpsRemoteOperationService git.GitOpsRemoteOperationService + gitOpsRemoteOperationService git.GitOperationService } func NewCdPipelineConfigServiceImpl(logger *zap.SugaredLogger, pipelineRepository pipelineConfig.PipelineRepository, @@ -169,7 +169,7 @@ func NewCdPipelineConfigServiceImpl(logger *zap.SugaredLogger, pipelineRepositor argoClientWrapperService argocdServer.ArgoClientWrapperService, deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService, gitOpsConfigReadService config.GitOpsConfigReadService, - gitOpsRemoteOperationService git.GitOpsRemoteOperationService) *CdPipelineConfigServiceImpl { + gitOpsRemoteOperationService git.GitOperationService) *CdPipelineConfigServiceImpl { return &CdPipelineConfigServiceImpl{ logger: logger, pipelineRepository: pipelineRepository, diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index 9b1aead59a..653e63ee65 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -193,7 +193,7 @@ type WorkflowDagExecutorImpl struct { deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService chartRefService chartRef.ChartRefService gitOpsConfigReadService config.GitOpsConfigReadService - gitOpsRemoteOperationService git.GitOpsRemoteOperationService + gitOpsRemoteOperationService git.GitOperationService } const kedaAutoscaling = "kedaAutoscaling" @@ -314,7 +314,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService, chartRefService chartRef.ChartRefService, gitOpsConfigReadService config.GitOpsConfigReadService, - gitOpsRemoteOperationService git.GitOpsRemoteOperationService) *WorkflowDagExecutorImpl { + gitOpsRemoteOperationService git.GitOperationService) *WorkflowDagExecutorImpl { wde := &WorkflowDagExecutorImpl{logger: Logger, pipelineRepository: pipelineRepository, cdWorkflowRepository: cdWorkflowRepository, diff --git a/wire_gen.go b/wire_gen.go index bcea4e49e5..4a8ed0becf 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -377,8 +377,8 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - gitOpsRemoteOperationServiceImpl := git.NewGitOpsRemoteOperationServiceImpl(sugaredLogger, gitFactory, gitOpsConfigReadServiceImpl, chartTemplateServiceImpl) - appServiceImpl := app2.NewAppService(envConfigOverrideRepositoryImpl, pipelineOverrideRepositoryImpl, mergeUtil, sugaredLogger, pipelineRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, applicationServiceClientImpl, appRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, chartTemplateServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, appStatusServiceImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, globalEnvVariables, scopedVariableCMCSManagerImpl, acdConfig, chartRefServiceImpl, gitOpsConfigReadServiceImpl, gitOpsRemoteOperationServiceImpl) + gitOperationServiceImpl := git.NewGitOperationServiceImpl(sugaredLogger, gitFactory, gitOpsConfigReadServiceImpl, chartTemplateServiceImpl) + appServiceImpl := app2.NewAppService(envConfigOverrideRepositoryImpl, pipelineOverrideRepositoryImpl, mergeUtil, sugaredLogger, pipelineRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, applicationServiceClientImpl, appRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, chartTemplateServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, appStatusServiceImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, globalEnvVariables, scopedVariableCMCSManagerImpl, acdConfig, chartRefServiceImpl, gitOpsConfigReadServiceImpl, gitOperationServiceImpl) validate, err := util.IntValidator() if err != nil { return nil, err @@ -437,7 +437,7 @@ func InitializeApp() (*App, error) { pipelineStrategyHistoryRepositoryImpl := repository8.NewPipelineStrategyHistoryRepositoryImpl(sugaredLogger, db) pipelineStrategyHistoryServiceImpl := history.NewPipelineStrategyHistoryServiceImpl(sugaredLogger, pipelineStrategyHistoryRepositoryImpl, userServiceImpl) manifestPushConfigRepositoryImpl := repository9.NewManifestPushConfigRepository(sugaredLogger, db) - gitOpsManifestPushServiceImpl := app2.NewGitOpsManifestPushServiceImpl(sugaredLogger, pipelineStatusTimelineServiceImpl, pipelineStatusTimelineRepositoryImpl, acdConfig, chartRefServiceImpl, gitOpsConfigReadServiceImpl, gitOpsRemoteOperationServiceImpl) + gitOpsManifestPushServiceImpl := app2.NewGitOpsManifestPushServiceImpl(sugaredLogger, pipelineStatusTimelineServiceImpl, pipelineStatusTimelineRepositoryImpl, acdConfig, chartRefServiceImpl, gitOpsConfigReadServiceImpl, gitOperationServiceImpl) imageScanHistoryRepositoryImpl := security.NewImageScanHistoryRepositoryImpl(db, sugaredLogger) imageScanDeployInfoRepositoryImpl := security.NewImageScanDeployInfoRepositoryImpl(db, sugaredLogger) genericNoteRepositoryImpl := repository12.NewGenericNoteRepositoryImpl(db) @@ -452,7 +452,7 @@ func InitializeApp() (*App, error) { repositoryServiceClientImpl := repository13.NewServiceClientImpl(sugaredLogger, argoCDConnectionManagerImpl) argoClientWrapperServiceImpl := argocdServer.NewArgoClientWrapperServiceImpl(sugaredLogger, applicationServiceClientImpl, acdConfig, repositoryServiceClientImpl) pipelineConfigListenerServiceImpl := pipeline.NewPipelineConfigListenerServiceImpl(sugaredLogger) - workflowDagExecutorImpl := pipeline.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, pubSubClientServiceImpl, appServiceImpl, workflowServiceImpl, ciArtifactRepositoryImpl, ciPipelineRepositoryImpl, materialRepositoryImpl, pipelineOverrideRepositoryImpl, userServiceImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, enforcerUtilImpl, eventSimpleFactoryImpl, eventRESTClientImpl, cvePolicyRepositoryImpl, imageScanResultRepositoryImpl, appWorkflowRepositoryImpl, prePostCdScriptHistoryServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineServiceImpl, ciTemplateRepositoryImpl, ciWorkflowRepositoryImpl, appLabelRepositoryImpl, clientImpl, pipelineStageServiceImpl, k8sCommonServiceImpl, globalPluginServiceImpl, pluginInputVariableParserImpl, scopedVariableCMCSManagerImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineStrategyHistoryServiceImpl, manifestPushConfigRepositoryImpl, gitOpsManifestPushServiceImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, appCrudOperationServiceImpl, pipelineConfigRepositoryImpl, dockerRegistryIpsConfigServiceImpl, chartRepositoryImpl, chartTemplateServiceImpl, pipelineStrategyHistoryRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, argoK8sClientImpl, configMapRepositoryImpl, configMapHistoryRepositoryImpl, helmAppServiceImpl, helmAppClientImpl, envConfigOverrideRepositoryImpl, mergeUtil, applicationServiceClientImpl, argoClientWrapperServiceImpl, pipelineConfigListenerServiceImpl, customTagServiceImpl, acdConfig, deployedAppMetricsServiceImpl, chartRefServiceImpl, gitOpsConfigReadServiceImpl, gitOpsRemoteOperationServiceImpl) + workflowDagExecutorImpl := pipeline.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, pubSubClientServiceImpl, appServiceImpl, workflowServiceImpl, ciArtifactRepositoryImpl, ciPipelineRepositoryImpl, materialRepositoryImpl, pipelineOverrideRepositoryImpl, userServiceImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, enforcerUtilImpl, eventSimpleFactoryImpl, eventRESTClientImpl, cvePolicyRepositoryImpl, imageScanResultRepositoryImpl, appWorkflowRepositoryImpl, prePostCdScriptHistoryServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineServiceImpl, ciTemplateRepositoryImpl, ciWorkflowRepositoryImpl, appLabelRepositoryImpl, clientImpl, pipelineStageServiceImpl, k8sCommonServiceImpl, globalPluginServiceImpl, pluginInputVariableParserImpl, scopedVariableCMCSManagerImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineStrategyHistoryServiceImpl, manifestPushConfigRepositoryImpl, gitOpsManifestPushServiceImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, appCrudOperationServiceImpl, pipelineConfigRepositoryImpl, dockerRegistryIpsConfigServiceImpl, chartRepositoryImpl, chartTemplateServiceImpl, pipelineStrategyHistoryRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, argoK8sClientImpl, configMapRepositoryImpl, configMapHistoryRepositoryImpl, helmAppServiceImpl, helmAppClientImpl, envConfigOverrideRepositoryImpl, mergeUtil, applicationServiceClientImpl, argoClientWrapperServiceImpl, pipelineConfigListenerServiceImpl, customTagServiceImpl, acdConfig, deployedAppMetricsServiceImpl, chartRefServiceImpl, gitOpsConfigReadServiceImpl, gitOperationServiceImpl) deploymentGroupAppRepositoryImpl := repository.NewDeploymentGroupAppRepositoryImpl(sugaredLogger, db) deploymentGroupServiceImpl := deploymentGroup.NewDeploymentGroupServiceImpl(appRepositoryImpl, sugaredLogger, pipelineRepositoryImpl, ciPipelineRepositoryImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, deploymentGroupAppRepositoryImpl, ciArtifactRepositoryImpl, appWorkflowRepositoryImpl, workflowDagExecutorImpl) deploymentConfigServiceImpl := pipeline.NewDeploymentConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, pipelineRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, configMapHistoryServiceImpl, scopedVariableCMCSManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) @@ -493,7 +493,7 @@ func InitializeApp() (*App, error) { return nil, err } devtronAppCMCSServiceImpl := pipeline.NewDevtronAppCMCSServiceImpl(sugaredLogger, appServiceImpl, attributesRepositoryImpl) - cdPipelineConfigServiceImpl := pipeline.NewCdPipelineConfigServiceImpl(sugaredLogger, pipelineRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, appRepositoryImpl, appServiceImpl, deploymentGroupRepositoryImpl, ciCdPipelineOrchestratorImpl, appStatusRepositoryImpl, ciPipelineRepositoryImpl, prePostCdScriptHistoryServiceImpl, clusterRepositoryImpl, helmAppServiceImpl, enforcerUtilImpl, pipelineStrategyHistoryServiceImpl, chartRepositoryImpl, resourceGroupServiceImpl, propertiesConfigServiceImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deploymentServiceTypeConfig, applicationServiceClientImpl, customTagServiceImpl, pipelineConfigListenerServiceImpl, devtronAppCMCSServiceImpl, ciPipelineConfigServiceImpl, buildPipelineSwitchServiceImpl, argoClientWrapperServiceImpl, deployedAppMetricsServiceImpl, gitOpsConfigReadServiceImpl, gitOpsRemoteOperationServiceImpl) + cdPipelineConfigServiceImpl := pipeline.NewCdPipelineConfigServiceImpl(sugaredLogger, pipelineRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, appRepositoryImpl, appServiceImpl, deploymentGroupRepositoryImpl, ciCdPipelineOrchestratorImpl, appStatusRepositoryImpl, ciPipelineRepositoryImpl, prePostCdScriptHistoryServiceImpl, clusterRepositoryImpl, helmAppServiceImpl, enforcerUtilImpl, pipelineStrategyHistoryServiceImpl, chartRepositoryImpl, resourceGroupServiceImpl, propertiesConfigServiceImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deploymentServiceTypeConfig, applicationServiceClientImpl, customTagServiceImpl, pipelineConfigListenerServiceImpl, devtronAppCMCSServiceImpl, ciPipelineConfigServiceImpl, buildPipelineSwitchServiceImpl, argoClientWrapperServiceImpl, deployedAppMetricsServiceImpl, gitOpsConfigReadServiceImpl, gitOperationServiceImpl) appArtifactManagerImpl := pipeline.NewAppArtifactManagerImpl(sugaredLogger, cdWorkflowRepositoryImpl, userServiceImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, ciWorkflowRepositoryImpl, pipelineStageServiceImpl, cdPipelineConfigServiceImpl, dockerArtifactStoreRepositoryImpl, ciPipelineRepositoryImpl, ciTemplateServiceImpl) globalStrategyMetadataChartRefMappingRepositoryImpl := chartRepoRepository.NewGlobalStrategyMetadataChartRefMappingRepositoryImpl(db, sugaredLogger) devtronAppStrategyServiceImpl := pipeline.NewDevtronAppStrategyServiceImpl(sugaredLogger, chartRepositoryImpl, globalStrategyMetadataChartRefMappingRepositoryImpl, ciCdPipelineOrchestratorImpl, cdPipelineConfigServiceImpl) @@ -538,16 +538,16 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - appStoreDeploymentCommonServiceImpl := appStoreDeploymentCommon.NewAppStoreDeploymentCommonServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, chartTemplateServiceImpl, gitFactory, gitOpsConfigReadServiceImpl, gitOpsRemoteOperationServiceImpl) - appStoreDeploymentFullModeServiceImpl := appStoreDeploymentFullMode.NewAppStoreDeploymentFullModeServiceImpl(sugaredLogger, applicationServiceClientImpl, argoK8sClientImpl, acdAuthConfig, argoUserServiceImpl, pipelineStatusTimelineServiceImpl, appStoreDeploymentCommonServiceImpl, argoClientWrapperServiceImpl, pubSubClientServiceImpl, installedAppVersionHistoryRepositoryImpl, acdConfig, gitOpsConfigReadServiceImpl, gitOpsRemoteOperationServiceImpl) + appStoreDeploymentCommonServiceImpl := appStoreDeploymentCommon.NewAppStoreDeploymentCommonServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, chartTemplateServiceImpl, gitFactory, gitOpsConfigReadServiceImpl, gitOperationServiceImpl) + appStoreDeploymentFullModeServiceImpl := appStoreDeploymentFullMode.NewAppStoreDeploymentFullModeServiceImpl(sugaredLogger, applicationServiceClientImpl, argoK8sClientImpl, acdAuthConfig, argoUserServiceImpl, pipelineStatusTimelineServiceImpl, appStoreDeploymentCommonServiceImpl, argoClientWrapperServiceImpl, pubSubClientServiceImpl, installedAppVersionHistoryRepositoryImpl, acdConfig, gitOpsConfigReadServiceImpl, gitOperationServiceImpl) clusterInstalledAppsRepositoryImpl := repository3.NewClusterInstalledAppsRepositoryImpl(db, sugaredLogger) - appStoreDeploymentHelmServiceImpl := appStoreDeploymentTool.NewAppStoreDeploymentHelmServiceImpl(sugaredLogger, helmAppServiceImpl, appStoreApplicationVersionRepositoryImpl, helmAppClientImpl, installedAppRepositoryImpl, appStoreDeploymentCommonServiceImpl, ociRegistryConfigRepositoryImpl, gitOpsRemoteOperationServiceImpl) - appStoreDeploymentArgoCdServiceImpl := appStoreDeploymentTool.NewAppStoreDeploymentArgoCdServiceImpl(sugaredLogger, appStoreDeploymentFullModeServiceImpl, applicationServiceClientImpl, chartGroupDeploymentRepositoryImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, appStoreDeploymentCommonServiceImpl, helmAppServiceImpl, appStatusServiceImpl, pipelineStatusTimelineServiceImpl, userServiceImpl, pipelineStatusTimelineRepositoryImpl, appStoreApplicationVersionRepositoryImpl, argoClientWrapperServiceImpl, acdConfig, gitOpsRemoteOperationServiceImpl) + appStoreDeploymentHelmServiceImpl := appStoreDeploymentTool.NewAppStoreDeploymentHelmServiceImpl(sugaredLogger, helmAppServiceImpl, appStoreApplicationVersionRepositoryImpl, helmAppClientImpl, installedAppRepositoryImpl, appStoreDeploymentCommonServiceImpl, ociRegistryConfigRepositoryImpl, gitOperationServiceImpl) + appStoreDeploymentArgoCdServiceImpl := appStoreDeploymentTool.NewAppStoreDeploymentArgoCdServiceImpl(sugaredLogger, appStoreDeploymentFullModeServiceImpl, applicationServiceClientImpl, chartGroupDeploymentRepositoryImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, appStoreDeploymentCommonServiceImpl, helmAppServiceImpl, appStatusServiceImpl, pipelineStatusTimelineServiceImpl, userServiceImpl, pipelineStatusTimelineRepositoryImpl, appStoreApplicationVersionRepositoryImpl, argoClientWrapperServiceImpl, acdConfig, gitOperationServiceImpl) serviceDeploymentServiceTypeConfig, err := service2.GetDeploymentServiceTypeConfig() if err != nil { return nil, err } - appStoreDeploymentServiceImpl := service2.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, clusterInstalledAppsRepositoryImpl, appRepositoryImpl, appStoreDeploymentHelmServiceImpl, appStoreDeploymentArgoCdServiceImpl, environmentServiceImpl, clusterServiceImplExtended, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, installedAppVersionHistoryRepositoryImpl, gitOpsConfigRepositoryImpl, serviceDeploymentServiceTypeConfig, acdConfig, gitOpsConfigReadServiceImpl, gitOpsRemoteOperationServiceImpl) + appStoreDeploymentServiceImpl := service2.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, clusterInstalledAppsRepositoryImpl, appRepositoryImpl, appStoreDeploymentHelmServiceImpl, appStoreDeploymentArgoCdServiceImpl, environmentServiceImpl, clusterServiceImplExtended, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, installedAppVersionHistoryRepositoryImpl, gitOpsConfigRepositoryImpl, serviceDeploymentServiceTypeConfig, acdConfig, gitOpsConfigReadServiceImpl, gitOperationServiceImpl) k8sResourceHistoryRepositoryImpl := repository16.NewK8sResourceHistoryRepositoryImpl(db, sugaredLogger) k8sResourceHistoryServiceImpl := kubernetesResourceAuditLogs.Newk8sResourceHistoryServiceImpl(k8sResourceHistoryRepositoryImpl, sugaredLogger, appRepositoryImpl, environmentRepositoryImpl) ephemeralContainersRepositoryImpl := repository2.NewEphemeralContainersRepositoryImpl(db) @@ -557,7 +557,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - installedAppServiceImpl, err := service2.NewInstalledAppServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, teamRepositoryImpl, appRepositoryImpl, applicationServiceClientImpl, appStoreValuesServiceImpl, pubSubClientServiceImpl, chartGroupDeploymentRepositoryImpl, environmentServiceImpl, acdAuthConfig, gitOpsConfigRepositoryImpl, userServiceImpl, appStoreDeploymentFullModeServiceImpl, appStoreDeploymentServiceImpl, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, helmAppClientImpl, helmAppServiceImpl, appStatusServiceImpl, k8sServiceImpl, pipelineStatusTimelineServiceImpl, appStoreDeploymentCommonServiceImpl, k8sCommonServiceImpl, k8sApplicationServiceImpl, acdConfig, gitOpsConfigReadServiceImpl, gitOpsRemoteOperationServiceImpl) + installedAppServiceImpl, err := service2.NewInstalledAppServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, teamRepositoryImpl, appRepositoryImpl, applicationServiceClientImpl, appStoreValuesServiceImpl, pubSubClientServiceImpl, chartGroupDeploymentRepositoryImpl, environmentServiceImpl, acdAuthConfig, gitOpsConfigRepositoryImpl, userServiceImpl, appStoreDeploymentFullModeServiceImpl, appStoreDeploymentServiceImpl, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, helmAppClientImpl, helmAppServiceImpl, appStatusServiceImpl, k8sServiceImpl, pipelineStatusTimelineServiceImpl, appStoreDeploymentCommonServiceImpl, k8sCommonServiceImpl, k8sApplicationServiceImpl, acdConfig, gitOpsConfigReadServiceImpl, gitOperationServiceImpl) if err != nil { return nil, err } From 9cb4432da827765deaaa92aebe8d97ae79917b1b Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 23 Jan 2024 13:21:35 +0530 Subject: [PATCH 33/64] migrated usages of gitService to gitOperationService --- pkg/app/AppService.go | 8 ++-- pkg/app/ManifestPushService.go | 12 ++--- .../common/AppStoreDeploymentCommonService.go | 45 +++++++++---------- .../AppStoreDeploymentFullModeService.go | 10 ++--- .../service/AppStoreDeploymentService.go | 12 ++--- .../deployment/service/InstalledAppService.go | 8 ++-- .../tool/AppStoreDeploymentArgoCdService.go | 8 ++-- .../tool/AppStoreDeploymentHelmService.go | 8 ++-- .../gitOps/git/GitOperationService.go | 35 +++++++++++++++ .../DeploymentPipelineConfigService.go | 8 ++-- pkg/pipeline/WorkflowDagExecutor.go | 6 +-- 11 files changed, 96 insertions(+), 64 deletions(-) diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index 34c10331e1..62a158d4af 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -122,7 +122,7 @@ type AppServiceImpl struct { acdConfig *argocdServer.ACDConfig chartRefService chartRef.ChartRefService gitOpsConfigReadService config.GitOpsConfigReadService - gitOpsRemoteOperationService git.GitOperationService + gitOperationService git.GitOperationService } type AppService interface { @@ -177,7 +177,7 @@ func NewAppService( scopedVariableManager variables.ScopedVariableCMCSManager, acdConfig *argocdServer.ACDConfig, chartRefService chartRef.ChartRefService, gitOpsConfigReadService config.GitOpsConfigReadService, - gitOpsRemoteOperationService git.GitOperationService) *AppServiceImpl { + gitOperationService git.GitOperationService) *AppServiceImpl { appServiceImpl := &AppServiceImpl{ environmentConfigRepository: environmentConfigRepository, mergeUtil: mergeUtil, @@ -207,7 +207,7 @@ func NewAppService( acdConfig: acdConfig, chartRefService: chartRefService, gitOpsConfigReadService: gitOpsConfigReadService, - gitOpsRemoteOperationService: gitOpsRemoteOperationService, + gitOperationService: gitOperationService, } return appServiceImpl } @@ -917,7 +917,7 @@ func (impl *AppServiceImpl) CreateGitopsRepo(app *app.App, userId int32) (gitops return "", nil, err } gitOpsRepoName := impl.gitOpsConfigReadService.GetGitOpsRepoName(app.AppName) - chartGitAttr, err = impl.gitOpsRemoteOperationService.CreateGitRepositoryForApp(gitOpsRepoName, chart.ReferenceTemplate, chart.ChartVersion, userId) + chartGitAttr, err = impl.gitOperationService.CreateGitRepositoryForApp(gitOpsRepoName, chart.ReferenceTemplate, chart.ChartVersion, userId) if err != nil { impl.logger.Errorw("error in pushing chart to git ", "gitOpsRepoName", gitOpsRepoName, "err", err) return "", nil, err diff --git a/pkg/app/ManifestPushService.go b/pkg/app/ManifestPushService.go index a3cc646729..50a1cc0af5 100644 --- a/pkg/app/ManifestPushService.go +++ b/pkg/app/ManifestPushService.go @@ -31,7 +31,7 @@ type GitOpsManifestPushServiceImpl struct { acdConfig *argocdServer.ACDConfig chartRefService chartRef.ChartRefService gitOpsConfigReadService config.GitOpsConfigReadService - gitOpsRemoteOperationService git.GitOperationService + gitOperationService git.GitOperationService } func NewGitOpsManifestPushServiceImpl(logger *zap.SugaredLogger, @@ -39,7 +39,7 @@ func NewGitOpsManifestPushServiceImpl(logger *zap.SugaredLogger, pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository, acdConfig *argocdServer.ACDConfig, chartRefService chartRef.ChartRefService, gitOpsConfigReadService config.GitOpsConfigReadService, - gitOpsRemoteOperationService git.GitOperationService) *GitOpsManifestPushServiceImpl { + gitOperationService git.GitOperationService) *GitOpsManifestPushServiceImpl { return &GitOpsManifestPushServiceImpl{ logger: logger, pipelineStatusTimelineService: pipelineStatusTimelineService, @@ -47,7 +47,7 @@ func NewGitOpsManifestPushServiceImpl(logger *zap.SugaredLogger, acdConfig: acdConfig, chartRefService: chartRefService, gitOpsConfigReadService: gitOpsConfigReadService, - gitOpsRemoteOperationService: gitOpsRemoteOperationService, + gitOperationService: gitOperationService, } } @@ -108,7 +108,7 @@ func (impl *GitOpsManifestPushServiceImpl) PushChartToGitRepo(manifestPushTempla impl.logger.Errorw("err in getting chart info", "err", err) return err } - err = impl.gitOpsRemoteOperationService.PushChartToGitRepo(gitOpsRepoName, manifestPushTemplate.ChartReferenceTemplate, manifestPushTemplate.ChartVersion, manifestPushTemplate.BuiltChartPath, manifestPushTemplate.RepoUrl, manifestPushTemplate.UserId) + err = impl.gitOperationService.PushChartToGitRepo(gitOpsRepoName, manifestPushTemplate.ChartReferenceTemplate, manifestPushTemplate.ChartVersion, manifestPushTemplate.BuiltChartPath, manifestPushTemplate.RepoUrl, manifestPushTemplate.UserId) if err != nil { impl.logger.Errorw("error in pushing chart to git", "err", err) return err @@ -135,8 +135,8 @@ func (impl *GitOpsManifestPushServiceImpl) CommitValuesToGit(manifestPushTemplat UserEmailId: userEmailId, } - _, span = otel.Tracer("orchestrator").Start(ctx, "gitOpsRemoteOperationService.CommitValues") - commitHash, commitTime, err = impl.gitOpsRemoteOperationService.CommitValues(chartGitAttr) + _, span = otel.Tracer("orchestrator").Start(ctx, "gitOperationService.CommitValues") + commitHash, commitTime, err = impl.gitOperationService.CommitValues(chartGitAttr) span.End() if err != nil { impl.logger.Errorw("error in git commit", "err", err) diff --git a/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go b/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go index 759ed4bd7a..ca443aeeef 100644 --- a/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go +++ b/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go @@ -69,7 +69,7 @@ type AppStoreDeploymentCommonServiceImpl struct { chartTemplateService util.ChartTemplateService gitFactory *util.GitFactory gitOpsConfigReadService config.GitOpsConfigReadService - gitOpsRemoteOperationService git.GitOperationService + gitOperationService git.GitOperationService } func NewAppStoreDeploymentCommonServiceImpl( @@ -80,7 +80,7 @@ func NewAppStoreDeploymentCommonServiceImpl( chartTemplateService util.ChartTemplateService, gitFactory *util.GitFactory, gitOpsConfigReadService config.GitOpsConfigReadService, - gitOpsRemoteOperationService git.GitOperationService) *AppStoreDeploymentCommonServiceImpl { + gitOperationService git.GitOperationService) *AppStoreDeploymentCommonServiceImpl { return &AppStoreDeploymentCommonServiceImpl{ logger: logger, installedAppRepository: installedAppRepository, @@ -89,7 +89,7 @@ func NewAppStoreDeploymentCommonServiceImpl( chartTemplateService: chartTemplateService, gitFactory: gitFactory, gitOpsConfigReadService: gitOpsConfigReadService, - gitOpsRemoteOperationService: gitOpsRemoteOperationService, + gitOperationService: gitOperationService, } } @@ -420,7 +420,7 @@ func (impl AppStoreDeploymentCommonServiceImpl) CreateGitOpsRepo(installAppVersi BitBucketWorkspaceId: bitbucketMetadata.BitBucketWorkspaceId, BitBucketProjectKey: bitbucketMetadata.BitBucketProjectKey, } - repoUrl, isNew, err := impl.gitOpsRemoteOperationService.CreateRepository(gitRepoRequest, installAppVersionRequest.UserId) + repoUrl, isNew, err := impl.gitOperationService.CreateRepository(gitRepoRequest, installAppVersionRequest.UserId) if err != nil { impl.logger.Errorw("error in creating git project", "name", installAppVersionRequest.GitOpsRepoName, "err", err) return "", false, err @@ -433,22 +433,19 @@ func (impl AppStoreDeploymentCommonServiceImpl) PushChartToGitopsRepo(PushChartT space := regexp.MustCompile(`\s+`) appStoreName := space.ReplaceAllString(PushChartToGitRequest.ChartAppStoreName, "-") chartDir := fmt.Sprintf("%s-%s", PushChartToGitRequest.AppName, impl.chartTemplateService.GetDir()) - clonedDir := impl.gitFactory.GitService.GetCloneDirectory(chartDir) - if _, err := os.Stat(clonedDir); os.IsNotExist(err) { - clonedDir, err = impl.gitFactory.GitService.Clone(PushChartToGitRequest.RepoURL, chartDir) - if err != nil { - impl.logger.Errorw("error in cloning repo", "url", PushChartToGitRequest.RepoURL, "err", err) - return nil, "", err - } - } else { - err = impl.gitOpsRemoteOperationService.GitPull(clonedDir, PushChartToGitRequest.RepoURL, appStoreName) - if err != nil { - return nil, "", err - } + clonedDir, err := impl.gitOperationService.GetClonedDir(chartDir, PushChartToGitRequest.RepoURL) + if err != nil { + impl.logger.Errorw("error in cloning repo", "url", PushChartToGitRequest.RepoURL, "err", err) + return nil, "", err + } + err = impl.gitOperationService.GitPull(clonedDir, PushChartToGitRequest.RepoURL, appStoreName) + if err != nil { + return nil, "", err } + acdAppName := fmt.Sprintf("%s-%s", PushChartToGitRequest.AppName, PushChartToGitRequest.EnvName) dir := filepath.Join(clonedDir, acdAppName) - err := os.MkdirAll(dir, os.ModePerm) + err = os.MkdirAll(dir, os.ModePerm) if err != nil { impl.logger.Errorw("error in making dir", "err", err) return nil, "", err @@ -469,11 +466,11 @@ func (impl AppStoreDeploymentCommonServiceImpl) PushChartToGitopsRepo(PushChartT return nil, "", err } userEmailId, userName := impl.gitOpsConfigReadService.GetUserEmailIdAndNameForGitOpsCommit(PushChartToGitRequest.UserId) - commit, err := impl.gitFactory.GitService.CommitAndPushAllChanges(clonedDir, "first commit", userName, userEmailId) + commit, err := impl.gitOperationService.CommitAndPushAllChanges(clonedDir, "first commit", userName, userEmailId) if err != nil { impl.logger.Errorw("error in pushing git", "err", err) impl.logger.Warn("re-trying, taking pull and then push again") - err = impl.gitOpsRemoteOperationService.GitPull(clonedDir, PushChartToGitRequest.RepoURL, acdAppName) + err = impl.gitOperationService.GitPull(clonedDir, PushChartToGitRequest.RepoURL, acdAppName) if err != nil { impl.logger.Errorw("error in git pull", "err", err, "appName", acdAppName) return nil, "", err @@ -483,7 +480,7 @@ func (impl AppStoreDeploymentCommonServiceImpl) PushChartToGitopsRepo(PushChartT impl.logger.Errorw("error copying dir", "err", err) return nil, "", err } - commit, err = impl.gitFactory.GitService.CommitAndPushAllChanges(clonedDir, "first commit", userName, userEmailId) + commit, err = impl.gitOperationService.CommitAndPushAllChanges(clonedDir, "first commit", userName, userEmailId) if err != nil { impl.logger.Errorw("error in pushing git", "err", err) return nil, "", err @@ -547,23 +544,23 @@ func (impl AppStoreDeploymentCommonServiceImpl) GitOpsOperations(manifestRespons // Checking this is the first time chart has been pushed , if yes requirements.yaml has been already pushed with chart as there was sync-delay with github api. // step-2 commit dependencies and values in git if !isNew { - _, _, err = impl.gitOpsRemoteOperationService.CommitValues(manifestResponse.RequirementsConfig) + _, _, err = impl.gitOperationService.CommitValues(manifestResponse.RequirementsConfig) if err != nil { impl.logger.Errorw("error in committing dependency config to git", "err", err) return appStoreGitOpsResponse, err } - err = impl.gitOpsRemoteOperationService.GitPull(clonedDir, chartGitAttribute.RepoUrl, appStoreName) + err = impl.gitOperationService.GitPull(clonedDir, chartGitAttribute.RepoUrl, appStoreName) if err != nil { impl.logger.Errorw("error in git pull", "err", err) return appStoreGitOpsResponse, err } - githash, _, err = impl.gitOpsRemoteOperationService.CommitValues(manifestResponse.ValuesConfig) + githash, _, err = impl.gitOperationService.CommitValues(manifestResponse.ValuesConfig) if err != nil { impl.logger.Errorw("error in committing values config to git", "err", err) return appStoreGitOpsResponse, err } - err = impl.gitOpsRemoteOperationService.GitPull(clonedDir, chartGitAttribute.RepoUrl, appStoreName) + err = impl.gitOperationService.GitPull(clonedDir, chartGitAttribute.RepoUrl, appStoreName) if err != nil { impl.logger.Errorw("error in git pull", "err", err) return appStoreGitOpsResponse, err diff --git a/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go b/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go index e0a3e066f3..ad2242066f 100644 --- a/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go +++ b/pkg/appStore/deployment/fullMode/AppStoreDeploymentFullModeService.go @@ -73,7 +73,7 @@ type AppStoreDeploymentFullModeServiceImpl struct { installedAppRepositoryHistory repository4.InstalledAppVersionHistoryRepository ACDConfig *argocdServer.ACDConfig gitOpsConfigReadService config.GitOpsConfigReadService - gitOpsRemoteOperationService git.GitOperationService + gitOperationService git.GitOperationService } func NewAppStoreDeploymentFullModeServiceImpl(logger *zap.SugaredLogger, @@ -86,7 +86,7 @@ func NewAppStoreDeploymentFullModeServiceImpl(logger *zap.SugaredLogger, installedAppRepositoryHistory repository4.InstalledAppVersionHistoryRepository, ACDConfig *argocdServer.ACDConfig, gitOpsConfigReadService config.GitOpsConfigReadService, - gitOpsRemoteOperationService git.GitOperationService) *AppStoreDeploymentFullModeServiceImpl { + gitOperationService git.GitOperationService) *AppStoreDeploymentFullModeServiceImpl { appStoreDeploymentFullModeServiceImpl := &AppStoreDeploymentFullModeServiceImpl{ logger: logger, acdClient: acdClient, @@ -100,7 +100,7 @@ func NewAppStoreDeploymentFullModeServiceImpl(logger *zap.SugaredLogger, installedAppRepositoryHistory: installedAppRepositoryHistory, ACDConfig: ACDConfig, gitOpsConfigReadService: gitOpsConfigReadService, - gitOpsRemoteOperationService: gitOpsRemoteOperationService, + gitOperationService: gitOperationService, } err := appStoreDeploymentFullModeServiceImpl.subscribeHelmInstallStatus() if err != nil { @@ -217,7 +217,7 @@ func (impl AppStoreDeploymentFullModeServiceImpl) UpdateValuesYaml(installAppVer impl.logger.Errorw("error in getting git commit config", "err", err) } - commitHash, _, err := impl.gitOpsRemoteOperationService.CommitValues(valuesGitConfig) + commitHash, _, err := impl.gitOperationService.CommitValues(valuesGitConfig) if err != nil { impl.logger.Errorw("error in git commit", "err", err) return installAppVersionRequest, errors.New(pipelineConfig.TIMELINE_STATUS_GIT_COMMIT_FAILED) @@ -241,7 +241,7 @@ func (impl AppStoreDeploymentFullModeServiceImpl) UpdateRequirementYaml(installA return err } - _, _, err = impl.gitOpsRemoteOperationService.CommitValues(requirementsGitConfig) + _, _, err = impl.gitOperationService.CommitValues(requirementsGitConfig) if err != nil { impl.logger.Errorw("error in values commit", "err", err) return errors.New(pipelineConfig.TIMELINE_STATUS_GIT_COMMIT_FAILED) diff --git a/pkg/appStore/deployment/service/AppStoreDeploymentService.go b/pkg/appStore/deployment/service/AppStoreDeploymentService.go index 39b2d05519..c8f4db9f2e 100644 --- a/pkg/appStore/deployment/service/AppStoreDeploymentService.go +++ b/pkg/appStore/deployment/service/AppStoreDeploymentService.go @@ -108,7 +108,7 @@ type AppStoreDeploymentServiceImpl struct { deploymentTypeConfig *DeploymentServiceTypeConfig aCDConfig *argocdServer.ACDConfig gitOpsConfigReadService config.GitOpsConfigReadService - gitOpsRemoteOperationService git.GitOperationService + gitOperationService git.GitOperationService } func NewAppStoreDeploymentServiceImpl(logger *zap.SugaredLogger, installedAppRepository repository.InstalledAppRepository, @@ -120,7 +120,7 @@ func NewAppStoreDeploymentServiceImpl(logger *zap.SugaredLogger, installedAppRep installedAppRepositoryHistory repository.InstalledAppVersionHistoryRepository, gitOpsRepository repository2.GitOpsConfigRepository, deploymentTypeConfig *DeploymentServiceTypeConfig, aCDConfig *argocdServer.ACDConfig, gitOpsConfigReadService config.GitOpsConfigReadService, - gitOpsRemoteOperationService git.GitOperationService) *AppStoreDeploymentServiceImpl { + gitOperationService git.GitOperationService) *AppStoreDeploymentServiceImpl { appStoreDeploymentServiceImpl := &AppStoreDeploymentServiceImpl{ logger: logger, @@ -141,7 +141,7 @@ func NewAppStoreDeploymentServiceImpl(logger *zap.SugaredLogger, installedAppRep deploymentTypeConfig: deploymentTypeConfig, aCDConfig: aCDConfig, gitOpsConfigReadService: gitOpsConfigReadService, - gitOpsRemoteOperationService: gitOpsRemoteOperationService, + gitOperationService: gitOperationService, } return appStoreDeploymentServiceImpl } @@ -1475,12 +1475,12 @@ func (impl *AppStoreDeploymentServiceImpl) UpdateInstalledApp(ctx context.Contex } else if isChartChanged || isVersionChanged { // update dependency if chart or chart version is changed - _, _, requirementsCommitErr = impl.gitOpsRemoteOperationService.CommitValues(manifest.RequirementsConfig) - gitHash, _, valuesCommitErr = impl.gitOpsRemoteOperationService.CommitValues(manifest.ValuesConfig) + _, _, requirementsCommitErr = impl.gitOperationService.CommitValues(manifest.RequirementsConfig) + gitHash, _, valuesCommitErr = impl.gitOperationService.CommitValues(manifest.ValuesConfig) } else { // only values are changed in update, so commit values config - gitHash, _, valuesCommitErr = impl.gitOpsRemoteOperationService.CommitValues(manifest.ValuesConfig) + gitHash, _, valuesCommitErr = impl.gitOperationService.CommitValues(manifest.ValuesConfig) } if valuesCommitErr != nil || requirementsCommitErr != nil { diff --git a/pkg/appStore/deployment/service/InstalledAppService.go b/pkg/appStore/deployment/service/InstalledAppService.go index 0bac817246..5566598f01 100644 --- a/pkg/appStore/deployment/service/InstalledAppService.go +++ b/pkg/appStore/deployment/service/InstalledAppService.go @@ -124,7 +124,7 @@ type InstalledAppServiceImpl struct { k8sApplicationService application3.K8sApplicationService acdConfig *argocdServer.ACDConfig gitOpsConfigReadService config.GitOpsConfigReadService - gitOpsRemoteOperationService git.GitOperationService + gitOperationService git.GitOperationService } func NewInstalledAppServiceImpl(logger *zap.SugaredLogger, @@ -147,7 +147,7 @@ func NewInstalledAppServiceImpl(logger *zap.SugaredLogger, appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService, k8sCommonService k8s.K8sCommonService, k8sApplicationService application3.K8sApplicationService, acdConfig *argocdServer.ACDConfig, gitOpsConfigReadService config.GitOpsConfigReadService, - gitOpsRemoteOperationService git.GitOperationService) (*InstalledAppServiceImpl, error) { + gitOperationService git.GitOperationService) (*InstalledAppServiceImpl, error) { impl := &InstalledAppServiceImpl{ logger: logger, installedAppRepository: installedAppRepository, @@ -177,7 +177,7 @@ func NewInstalledAppServiceImpl(logger *zap.SugaredLogger, k8sApplicationService: k8sApplicationService, acdConfig: acdConfig, gitOpsConfigReadService: gitOpsConfigReadService, - gitOpsRemoteOperationService: gitOpsRemoteOperationService, + gitOperationService: gitOperationService, } err := impl.Subscribe() if err != nil { @@ -446,7 +446,7 @@ func (impl InstalledAppServiceImpl) performDeployStageOnAcd(installedAppVersion return nil, err } - repoUrl, err := impl.gitOpsRemoteOperationService.GetRepoUrlByRepoName(installedAppVersion.GitOpsRepoName) + repoUrl, err := impl.gitOperationService.GetRepoUrlByRepoName(installedAppVersion.GitOpsRepoName) if err != nil { //will allow to continue to persist status on next operation impl.logger.Errorw("error, GetRepoUrlByRepoName", "err", err) diff --git a/pkg/appStore/deployment/tool/AppStoreDeploymentArgoCdService.go b/pkg/appStore/deployment/tool/AppStoreDeploymentArgoCdService.go index a19fa6c975..501693cffd 100644 --- a/pkg/appStore/deployment/tool/AppStoreDeploymentArgoCdService.go +++ b/pkg/appStore/deployment/tool/AppStoreDeploymentArgoCdService.go @@ -71,7 +71,7 @@ type AppStoreDeploymentArgoCdServiceImpl struct { appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository argoClientWrapperService argocdServer.ArgoClientWrapperService acdConfig *argocdServer.ACDConfig - gitOpsRemoteOperationService git.GitOperationService + gitOperationService git.GitOperationService } func NewAppStoreDeploymentArgoCdServiceImpl(logger *zap.SugaredLogger, appStoreDeploymentFullModeService appStoreDeploymentFullMode.AppStoreDeploymentFullModeService, @@ -83,7 +83,7 @@ func NewAppStoreDeploymentArgoCdServiceImpl(logger *zap.SugaredLogger, appStoreD pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository, appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository, argoClientWrapperService argocdServer.ArgoClientWrapperService, acdConfig *argocdServer.ACDConfig, - gitOpsRemoteOperationService git.GitOperationService) *AppStoreDeploymentArgoCdServiceImpl { + gitOperationService git.GitOperationService) *AppStoreDeploymentArgoCdServiceImpl { return &AppStoreDeploymentArgoCdServiceImpl{ Logger: logger, appStoreDeploymentFullModeService: appStoreDeploymentFullModeService, @@ -101,7 +101,7 @@ func NewAppStoreDeploymentArgoCdServiceImpl(logger *zap.SugaredLogger, appStoreD appStoreApplicationVersionRepository: appStoreApplicationVersionRepository, argoClientWrapperService: argoClientWrapperService, acdConfig: acdConfig, - gitOpsRemoteOperationService: gitOpsRemoteOperationService, + gitOperationService: gitOperationService, } } @@ -544,7 +544,7 @@ func (impl AppStoreDeploymentArgoCdServiceImpl) updateValuesYaml(installAppVersi impl.Logger.Errorw("error in getting git config for helm app", "err", err) return nil, err } - gitHash, _, err := impl.gitOpsRemoteOperationService.CommitValues(valuesGitConfig) + gitHash, _, err := impl.gitOperationService.CommitValues(valuesGitConfig) if err != nil { impl.Logger.Errorw("error in git commit", "err", err) _ = impl.SaveTimelineForACDHelmApps(installAppVersionRequest, pipelineConfig.TIMELINE_STATUS_GIT_COMMIT_FAILED, fmt.Sprintf("Git commit failed - %v", err), time.Now(), tx) diff --git a/pkg/appStore/deployment/tool/AppStoreDeploymentHelmService.go b/pkg/appStore/deployment/tool/AppStoreDeploymentHelmService.go index d8dbfc167a..e57416218c 100644 --- a/pkg/appStore/deployment/tool/AppStoreDeploymentHelmService.go +++ b/pkg/appStore/deployment/tool/AppStoreDeploymentHelmService.go @@ -39,7 +39,7 @@ type AppStoreDeploymentHelmServiceImpl struct { installedAppRepository repository.InstalledAppRepository appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService OCIRegistryConfigRepository repository2.OCIRegistryConfigRepository - gitOpsRemoteOperationService git.GitOperationService + gitOperationService git.GitOperationService } func NewAppStoreDeploymentHelmServiceImpl(logger *zap.SugaredLogger, helmAppService client.HelmAppService, @@ -47,7 +47,7 @@ func NewAppStoreDeploymentHelmServiceImpl(logger *zap.SugaredLogger, helmAppServ helmAppClient client.HelmAppClient, installedAppRepository repository.InstalledAppRepository, appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService, OCIRegistryConfigRepository repository2.OCIRegistryConfigRepository, - gitOpsRemoteOperationService git.GitOperationService) *AppStoreDeploymentHelmServiceImpl { + gitOperationService git.GitOperationService) *AppStoreDeploymentHelmServiceImpl { return &AppStoreDeploymentHelmServiceImpl{ Logger: logger, helmAppService: helmAppService, @@ -56,7 +56,7 @@ func NewAppStoreDeploymentHelmServiceImpl(logger *zap.SugaredLogger, helmAppServ installedAppRepository: installedAppRepository, appStoreDeploymentCommonService: appStoreDeploymentCommonService, OCIRegistryConfigRepository: OCIRegistryConfigRepository, - gitOpsRemoteOperationService: gitOpsRemoteOperationService, + gitOperationService: gitOperationService, } } @@ -274,7 +274,7 @@ func (impl *AppStoreDeploymentHelmServiceImpl) UpdateValuesDependencies(installA impl.Logger.Errorw("error in getting git config for helm app", "err", err) return err } - _, _, err = impl.gitOpsRemoteOperationService.CommitValues(valuesGitConfig) + _, _, err = impl.gitOperationService.CommitValues(valuesGitConfig) if err != nil { impl.Logger.Errorw("error in committing config to git for helm app", "err", err) return err diff --git a/pkg/deployment/gitOps/git/GitOperationService.go b/pkg/deployment/gitOps/git/GitOperationService.go index e434666a51..08adb97933 100644 --- a/pkg/deployment/gitOps/git/GitOperationService.go +++ b/pkg/deployment/gitOps/git/GitOperationService.go @@ -26,9 +26,15 @@ type GitOperationService interface { CreateChartProxy(chartMetaData *chart.Metadata, refChartLocation string, envName string, chartProxyReq *bean.ChartProxyReqDto) (string, *commonBean.ChartGitAttribute, error) GitPull(clonedDir string, repoUrl string, appStoreName string) error + CommitValues(chartGitAttr *util.ChartConfig) (commitHash string, commitTime time.Time, err error) + CommitAndPushAllChanges(clonedDir, commitMsg, userName, userEmailId string) (commitHash string, err error) + CreateRepository(dto *bean2.GitOpsConfigDto, userId int32) (string, bool, error) GetRepoUrlByRepoName(repoName string) (string, error) + + GetClonedDir(chartDir, repoUrl string) (string, error) + CloneDir(repoUrl, chartDir string) (string, error) } type GitOperationServiceImpl struct { @@ -338,6 +344,15 @@ func (impl *GitOperationServiceImpl) CommitValues(chartGitAttr *util.ChartConfig return commitHash, commitTime, nil } +func (impl *GitOperationServiceImpl) CommitAndPushAllChanges(clonedDir, commitMsg, userName, userEmailId string) (commitHash string, err error) { + commitHash, err = impl.gitFactory.GitService.CommitAndPushAllChanges(clonedDir, commitMsg, userName, userEmailId) + if err != nil { + impl.logger.Errorw("error in pushing git", "err", err) + return commitHash, err + } + return commitHash, nil +} + func (impl *GitOperationServiceImpl) CreateRepository(dto *bean2.GitOpsConfigDto, userId int32) (string, bool, error) { //getting user name & emailId for commit author data userEmailId, userName := impl.gitOpsConfigReadService.GetUserEmailIdAndNameForGitOpsCommit(userId) @@ -374,3 +389,23 @@ func (impl *GitOperationServiceImpl) GetRepoUrlByRepoName(repoName string) (stri } return repoUrl, nil } + +func (impl *GitOperationServiceImpl) GetClonedDir(chartDir, repoUrl string) (string, error) { + clonedDir := impl.gitFactory.GitService.GetCloneDirectory(chartDir) + if _, err := os.Stat(clonedDir); os.IsNotExist(err) { + return impl.CloneDir(repoUrl, chartDir) + } else if err != nil { + impl.logger.Errorw("error in cloning repo", "url", repoUrl, "err", err) + return "", err + } + return clonedDir, nil +} + +func (impl *GitOperationServiceImpl) CloneDir(repoUrl, chartDir string) (string, error) { + clonedDir, err := impl.gitFactory.GitService.Clone(repoUrl, chartDir) + if err != nil { + impl.logger.Errorw("error in cloning repo", "url", repoUrl, "err", err) + return "", err + } + return clonedDir, nil +} diff --git a/pkg/pipeline/DeploymentPipelineConfigService.go b/pkg/pipeline/DeploymentPipelineConfigService.go index 72a15b312c..c806a1942e 100644 --- a/pkg/pipeline/DeploymentPipelineConfigService.go +++ b/pkg/pipeline/DeploymentPipelineConfigService.go @@ -148,7 +148,7 @@ type CdPipelineConfigServiceImpl struct { argoClientWrapperService argocdServer.ArgoClientWrapperService deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService gitOpsConfigReadService config.GitOpsConfigReadService - gitOpsRemoteOperationService git.GitOperationService + gitOperationService git.GitOperationService } func NewCdPipelineConfigServiceImpl(logger *zap.SugaredLogger, pipelineRepository pipelineConfig.PipelineRepository, @@ -169,7 +169,7 @@ func NewCdPipelineConfigServiceImpl(logger *zap.SugaredLogger, pipelineRepositor argoClientWrapperService argocdServer.ArgoClientWrapperService, deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService, gitOpsConfigReadService config.GitOpsConfigReadService, - gitOpsRemoteOperationService git.GitOperationService) *CdPipelineConfigServiceImpl { + gitOperationService git.GitOperationService) *CdPipelineConfigServiceImpl { return &CdPipelineConfigServiceImpl{ logger: logger, pipelineRepository: pipelineRepository, @@ -203,7 +203,7 @@ func NewCdPipelineConfigServiceImpl(logger *zap.SugaredLogger, pipelineRepositor argoClientWrapperService: argoClientWrapperService, deployedAppMetricsService: deployedAppMetricsService, gitOpsConfigReadService: gitOpsConfigReadService, - gitOpsRemoteOperationService: gitOpsRemoteOperationService, + gitOperationService: gitOperationService, } } @@ -1596,7 +1596,7 @@ func (impl *CdPipelineConfigServiceImpl) RegisterInACD(gitOpsRepoName string, ch emptyRepoErrorMessage := []string{"failed to get index: 404 Not Found", "git repository is empty"} if strings.Contains(err.Error(), emptyRepoErrorMessage[0]) || strings.Contains(err.Error(), emptyRepoErrorMessage[1]) { // - found empty repository, create some file in repository - err := impl.gitOpsRemoteOperationService.CreateReadmeInGitRepo(gitOpsRepoName, userId) + err := impl.gitOperationService.CreateReadmeInGitRepo(gitOpsRepoName, userId) if err != nil { impl.logger.Errorw("error in creating file in git repo", "err", err) return err diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index 653e63ee65..b8e404af3b 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -193,7 +193,7 @@ type WorkflowDagExecutorImpl struct { deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService chartRefService chartRef.ChartRefService gitOpsConfigReadService config.GitOpsConfigReadService - gitOpsRemoteOperationService git.GitOperationService + gitOperationService git.GitOperationService } const kedaAutoscaling = "kedaAutoscaling" @@ -314,7 +314,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService, chartRefService chartRef.ChartRefService, gitOpsConfigReadService config.GitOpsConfigReadService, - gitOpsRemoteOperationService git.GitOperationService) *WorkflowDagExecutorImpl { + gitOperationService git.GitOperationService) *WorkflowDagExecutorImpl { wde := &WorkflowDagExecutorImpl{logger: Logger, pipelineRepository: pipelineRepository, cdWorkflowRepository: cdWorkflowRepository, @@ -382,7 +382,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi deployedAppMetricsService: deployedAppMetricsService, chartRefService: chartRefService, gitOpsConfigReadService: gitOpsConfigReadService, - gitOpsRemoteOperationService: gitOpsRemoteOperationService, + gitOperationService: gitOperationService, } config, err := types.GetCdConfig() if err != nil { From c1adf3234b52b1662eee3ca327bae65906be4363 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 23 Jan 2024 13:29:43 +0530 Subject: [PATCH 34/64] shifted git service and all gitOps clients to pkg --- Wire.go | 5 ++-- cmd/external-app/wire.go | 5 ++-- cmd/external-app/wire_gen.go | 4 +-- internal/util/GitService_test.go | 9 +++--- pkg/app/ManifestPushService.go | 3 +- .../common/AppStoreDeploymentCommonService.go | 28 +++++++++---------- .../service/InstalledAppService_test.go | 3 +- .../deployment/gitOps/git}/GitCliUtil.go | 2 +- .../gitOps/git/GitOperationService.go | 8 +++--- .../deployment/gitOps/git}/GitService.go | 16 +++++------ .../deployment/gitOps/git}/GitServiceAzure.go | 4 +-- .../gitOps/git}/GitServiceBitbucket.go | 4 +-- .../gitOps/git}/GitServiceGithub.go | 4 +-- .../gitOps/git}/GitServiceGitlab.go | 2 +- pkg/gitops/GitOpsConfigService.go | 15 +++++----- wire_gen.go | 4 +-- 16 files changed, 60 insertions(+), 56 deletions(-) rename {internal/util => pkg/deployment/gitOps/git}/GitCliUtil.go (99%) rename {internal/util => pkg/deployment/gitOps/git}/GitService.go (96%) rename {internal/util => pkg/deployment/gitOps/git}/GitServiceAzure.go (99%) rename {internal/util => pkg/deployment/gitOps/git}/GitServiceBitbucket.go (99%) rename {internal/util => pkg/deployment/gitOps/git}/GitServiceGithub.go (99%) rename {internal/util => pkg/deployment/gitOps/git}/GitServiceGitlab.go (99%) diff --git a/Wire.go b/Wire.go index ef2877347b..d3b14ac7cb 100644 --- a/Wire.go +++ b/Wire.go @@ -100,6 +100,7 @@ import ( "github.com/devtron-labs/devtron/pkg/commonService" delete2 "github.com/devtron-labs/devtron/pkg/delete" deployment2 "github.com/devtron-labs/devtron/pkg/deployment" + git2 "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" "github.com/devtron-labs/devtron/pkg/deploymentGroup" "github.com/devtron-labs/devtron/pkg/devtronResource" repository9 "github.com/devtron-labs/devtron/pkg/devtronResource/repository" @@ -345,7 +346,7 @@ func InitializeApp() (*App, error) { wire.Bind(new(pipelineConfig.CiPipelineRepository), new(*pipelineConfig.CiPipelineRepositoryImpl)), pipelineConfig.NewCiPipelineMaterialRepositoryImpl, wire.Bind(new(pipelineConfig.CiPipelineMaterialRepository), new(*pipelineConfig.CiPipelineMaterialRepositoryImpl)), - util.NewGitFactory, + git2.NewGitFactory, application.NewApplicationClientImpl, wire.Bind(new(application.ServiceClient), new(*application.ServiceClientImpl)), @@ -658,7 +659,7 @@ func InitializeApp() (*App, error) { scopedVariable.NewScopedVariableRestHandlerImpl, wire.Bind(new(scopedVariable.ScopedVariableRestHandler), new(*scopedVariable.ScopedVariableRestHandlerImpl)), - util.NewGitCliUtil, + git2.NewGitCliUtil, router.NewTelemetryRouterImpl, wire.Bind(new(router.TelemetryRouter), new(*router.TelemetryRouterImpl)), diff --git a/cmd/external-app/wire.go b/cmd/external-app/wire.go index 01427bafe1..da0f16094b 100644 --- a/cmd/external-app/wire.go +++ b/cmd/external-app/wire.go @@ -44,6 +44,7 @@ import ( "github.com/devtron-labs/devtron/pkg/attributes" delete2 "github.com/devtron-labs/devtron/pkg/delete" "github.com/devtron-labs/devtron/pkg/deployment/gitOps" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" "github.com/devtron-labs/devtron/pkg/kubernetesResourceAuditLogs" repository2 "github.com/devtron-labs/devtron/pkg/kubernetesResourceAuditLogs/repository" "github.com/devtron-labs/devtron/pkg/pipeline" @@ -178,8 +179,8 @@ func InitializeApp() (*App, error) { util.NewChartTemplateServiceImpl, wire.Bind(new(util.ChartTemplateService), new(*util.ChartTemplateServiceImpl)), - util.NewGitFactory, - util.NewGitCliUtil, + git.NewGitFactory, + git.NewGitCliUtil, security2.NewScanToolMetadataRepositoryImpl, wire.Bind(new(security2.ScanToolMetadataRepository), new(*security2.ScanToolMetadataRepositoryImpl)), diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index 1f768a004f..dd3932c80d 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -234,8 +234,8 @@ func InitializeApp() (*App, error) { dashboardRouterImpl := dashboard.NewDashboardRouterImpl(sugaredLogger, dashboardConfig) chartTemplateServiceImpl := util.NewChartTemplateServiceImpl(sugaredLogger) gitOpsConfigRepositoryImpl := repository3.NewGitOpsConfigRepositoryImpl(sugaredLogger, db) - gitCliUtil := util.NewGitCliUtil(sugaredLogger) - gitFactory, err := util.NewGitFactory(sugaredLogger, gitOpsConfigRepositoryImpl, gitCliUtil) + gitCliUtil := git.NewGitCliUtil(sugaredLogger) + gitFactory, err := git.NewGitFactory(sugaredLogger, gitOpsConfigRepositoryImpl, gitCliUtil) if err != nil { return nil, err } diff --git a/internal/util/GitService_test.go b/internal/util/GitService_test.go index 6c23a4202d..10eee631a3 100644 --- a/internal/util/GitService_test.go +++ b/internal/util/GitService_test.go @@ -2,15 +2,16 @@ package util import ( "github.com/devtron-labs/devtron/api/bean" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" "testing" ) -func getTestGithubClient() GitHubClient { +func getTestGithubClient() git.GitHubClient { logger, err := NewSugardLogger() - gitCliUtl := NewGitCliUtil(logger) - gitService := NewGitServiceImpl(&GitConfig{GitToken: "", GitUserName: "nishant"}, logger, gitCliUtl) + gitCliUtl := git.NewGitCliUtil(logger) + gitService := git.NewGitServiceImpl(&git.GitConfig{GitToken: "", GitUserName: "nishant"}, logger, gitCliUtl) - githubClient, err := NewGithubClient("", "", "test-org", logger, gitService, nil) + githubClient, err := git.NewGithubClient("", "", "test-org", logger, gitService, nil) if err != nil { panic(err) } diff --git a/pkg/app/ManifestPushService.go b/pkg/app/ManifestPushService.go index 50a1cc0af5..6502a797d8 100644 --- a/pkg/app/ManifestPushService.go +++ b/pkg/app/ManifestPushService.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/devtron-labs/devtron/client/argocdServer" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app/bean" status2 "github.com/devtron-labs/devtron/pkg/app/status" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" @@ -124,7 +123,7 @@ func (impl *GitOpsManifestPushServiceImpl) CommitValuesToGit(manifestPushTemplat //getting username & emailId for commit author data userEmailId, userName := impl.gitOpsConfigReadService.GetUserEmailIdAndNameForGitOpsCommit(manifestPushTemplate.UserId) span.End() - chartGitAttr := &util.ChartConfig{ + chartGitAttr := &git.ChartConfig{ FileName: fmt.Sprintf("_%d-values.yaml", manifestPushTemplate.TargetEnvironmentName), FileContent: string(manifestPushTemplate.MergedValues), ChartName: manifestPushTemplate.ChartName, diff --git a/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go b/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go index ca443aeeef..68146304c0 100644 --- a/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go +++ b/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go @@ -50,10 +50,10 @@ type AppStoreDeploymentCommonService interface { GetInstalledAppByClusterNamespaceAndName(clusterId int, namespace string, appName string) (*appStoreBean.InstallAppVersionDTO, error) GetInstalledAppByInstalledAppId(installedAppId int) (*appStoreBean.InstallAppVersionDTO, error) ParseGitRepoErrorResponse(err error) (bool, error) - GetValuesAndRequirementGitConfig(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*util.ChartConfig, *util.ChartConfig, error) + GetValuesAndRequirementGitConfig(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*git.ChartConfig, *git.ChartConfig, error) CreateChartProxyAndGetPath(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*util.ChartCreateResponse, error) - CreateGitOpsRepoAndPushChart(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, builtChartPath string, requirementsConfig *util.ChartConfig, valuesConfig *util.ChartConfig) (*commonBean.ChartGitAttribute, bool, string, error) - GetGitCommitConfig(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, fileString string, filename string) (*util.ChartConfig, error) + CreateGitOpsRepoAndPushChart(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, builtChartPath string, requirementsConfig *git.ChartConfig, valuesConfig *git.ChartConfig) (*commonBean.ChartGitAttribute, bool, string, error) + GetGitCommitConfig(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, fileString string, filename string) (*git.ChartConfig, error) GetValuesString(chartName, valuesOverrideYaml string) (string, error) GetRequirementsString(appStoreVersionId int) (string, error) GenerateManifest(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (manifestResponse *AppStoreManifestResponse, err error) @@ -67,7 +67,7 @@ type AppStoreDeploymentCommonServiceImpl struct { appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository environmentRepository repository2.EnvironmentRepository chartTemplateService util.ChartTemplateService - gitFactory *util.GitFactory + gitFactory *git.GitFactory gitOpsConfigReadService config.GitOpsConfigReadService gitOperationService git.GitOperationService } @@ -78,7 +78,7 @@ func NewAppStoreDeploymentCommonServiceImpl( appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository, environmentRepository repository2.EnvironmentRepository, chartTemplateService util.ChartTemplateService, - gitFactory *util.GitFactory, + gitFactory *git.GitFactory, gitOpsConfigReadService config.GitOpsConfigReadService, gitOperationService git.GitOperationService) *AppStoreDeploymentCommonServiceImpl { return &AppStoreDeploymentCommonServiceImpl{ @@ -115,8 +115,8 @@ func ParseChartGitPushRequest(installAppRequestDTO *appStoreBean.InstallAppVersi type AppStoreManifestResponse struct { ChartResponse *util.ChartCreateResponse - ValuesConfig *util.ChartConfig - RequirementsConfig *util.ChartConfig + ValuesConfig *git.ChartConfig + RequirementsConfig *git.ChartConfig } type AppStoreGitOpsResponse struct { @@ -228,7 +228,7 @@ func (impl AppStoreDeploymentCommonServiceImpl) ParseGitRepoErrorResponse(err er impl.logger.Errorw("no content found while updating git repo gitlab, do auto fix", "error", err) noTargetFound = true } - if err.Error() == util.BITBUCKET_REPO_NOT_FOUND_ERROR { + if err.Error() == git.BITBUCKET_REPO_NOT_FOUND_ERROR { impl.logger.Errorw("no content found while updating git repo bitbucket, do auto fix", "error", err) noTargetFound = true } @@ -289,7 +289,7 @@ func (impl AppStoreDeploymentCommonServiceImpl) GetRequirementsString(appStoreVe return string(requirementDependenciesByte), nil } -func (impl AppStoreDeploymentCommonServiceImpl) GetGitCommitConfig(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, fileString string, filename string) (*util.ChartConfig, error) { +func (impl AppStoreDeploymentCommonServiceImpl) GetGitCommitConfig(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, fileString string, filename string) (*git.ChartConfig, error) { appStoreAppVersion, err := impl.appStoreApplicationVersionRepository.FindById(installAppVersionRequest.AppStoreVersion) if err != nil { impl.logger.Errorw("fetching error", "err", err) @@ -304,7 +304,7 @@ func (impl AppStoreDeploymentCommonServiceImpl) GetGitCommitConfig(installAppVer argocdAppName := installAppVersionRequest.AppName + "-" + environment.Name gitOpsRepoName := impl.gitOpsConfigReadService.GetGitOpsRepoName(installAppVersionRequest.AppName) userEmailId, userName := impl.gitOpsConfigReadService.GetUserEmailIdAndNameForGitOpsCommit(installAppVersionRequest.UserId) - YamlConfig := &util.ChartConfig{ + YamlConfig := &git.ChartConfig{ FileName: filename, FileContent: fileString, ChartName: installAppVersionRequest.AppName, @@ -317,7 +317,7 @@ func (impl AppStoreDeploymentCommonServiceImpl) GetGitCommitConfig(installAppVer return YamlConfig, nil } -func (impl AppStoreDeploymentCommonServiceImpl) GetValuesAndRequirementGitConfig(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*util.ChartConfig, *util.ChartConfig, error) { +func (impl AppStoreDeploymentCommonServiceImpl) GetValuesAndRequirementGitConfig(installAppVersionRequest *appStoreBean.InstallAppVersionDTO) (*git.ChartConfig, *git.ChartConfig, error) { appStoreAppVersion, err := impl.appStoreApplicationVersionRepository.FindById(installAppVersionRequest.AppStoreVersion) if err != nil { @@ -429,7 +429,7 @@ func (impl AppStoreDeploymentCommonServiceImpl) CreateGitOpsRepo(installAppVersi } // PushChartToGitopsRepo pushes built chart to gitOps repo -func (impl AppStoreDeploymentCommonServiceImpl) PushChartToGitopsRepo(PushChartToGitRequest *appStoreBean.PushChartToGitRequestDTO, requirementsConfig *util.ChartConfig, valuesConfig *util.ChartConfig) (*commonBean.ChartGitAttribute, string, error) { +func (impl AppStoreDeploymentCommonServiceImpl) PushChartToGitopsRepo(PushChartToGitRequest *appStoreBean.PushChartToGitRequestDTO, requirementsConfig *git.ChartConfig, valuesConfig *git.ChartConfig) (*commonBean.ChartGitAttribute, string, error) { space := regexp.MustCompile(`\s+`) appStoreName := space.ReplaceAllString(PushChartToGitRequest.ChartAppStoreName, "-") chartDir := fmt.Sprintf("%s-%s", PushChartToGitRequest.AppName, impl.chartTemplateService.GetDir()) @@ -492,7 +492,7 @@ func (impl AppStoreDeploymentCommonServiceImpl) PushChartToGitopsRepo(PushChartT } // AddConfigFileToChart will override requirements.yaml file in chart -func (impl AppStoreDeploymentCommonServiceImpl) AddConfigFileToChart(config *util.ChartConfig, dir string, clonedDir string) error { +func (impl AppStoreDeploymentCommonServiceImpl) AddConfigFileToChart(config *git.ChartConfig, dir string, clonedDir string) error { filePath := filepath.Join(clonedDir, config.FileName) file, err := os.Create(filePath) if err != nil { @@ -515,7 +515,7 @@ func (impl AppStoreDeploymentCommonServiceImpl) AddConfigFileToChart(config *uti } // CreateGitOpsRepoAndPushChart is a wrapper for creating gitops repo and pushing chart to created repo -func (impl AppStoreDeploymentCommonServiceImpl) CreateGitOpsRepoAndPushChart(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, builtChartPath string, requirementsConfig *util.ChartConfig, valuesConfig *util.ChartConfig) (*commonBean.ChartGitAttribute, bool, string, error) { +func (impl AppStoreDeploymentCommonServiceImpl) CreateGitOpsRepoAndPushChart(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, builtChartPath string, requirementsConfig *git.ChartConfig, valuesConfig *git.ChartConfig) (*commonBean.ChartGitAttribute, bool, string, error) { repoURL, isNew, err := impl.CreateGitOpsRepo(installAppVersionRequest) if err != nil { impl.logger.Errorw("Error in creating gitops repo for ", "appName", installAppVersionRequest.AppName, "err", err) diff --git a/pkg/appStore/deployment/service/InstalledAppService_test.go b/pkg/appStore/deployment/service/InstalledAppService_test.go index d942bbc4f5..2c2f90e7be 100644 --- a/pkg/appStore/deployment/service/InstalledAppService_test.go +++ b/pkg/appStore/deployment/service/InstalledAppService_test.go @@ -1,6 +1,7 @@ package service import ( + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" "testing" pubsub "github.com/devtron-labs/common-lib/pubsub-lib" @@ -40,7 +41,7 @@ func TestInstalledAppServiceImpl_DeployDefaultChartOnCluster(t *testing.T) { chartGroupDeploymentRepository repository5.ChartGroupDeploymentRepository envService cluster.EnvironmentService ArgoK8sClient argocdServer.ArgoK8sClient - gitFactory *util.GitFactory + gitFactory *git.GitFactory aCDAuthConfig *util2.ACDAuthConfig gitOpsRepository repository3.GitOpsConfigRepository userService user.UserService diff --git a/internal/util/GitCliUtil.go b/pkg/deployment/gitOps/git/GitCliUtil.go similarity index 99% rename from internal/util/GitCliUtil.go rename to pkg/deployment/gitOps/git/GitCliUtil.go index 54d9281a2a..587e20f1c6 100644 --- a/internal/util/GitCliUtil.go +++ b/pkg/deployment/gitOps/git/GitCliUtil.go @@ -1,4 +1,4 @@ -package util +package git import ( "fmt" diff --git a/pkg/deployment/gitOps/git/GitOperationService.go b/pkg/deployment/gitOps/git/GitOperationService.go index 08adb97933..6fa5f879d4 100644 --- a/pkg/deployment/gitOps/git/GitOperationService.go +++ b/pkg/deployment/gitOps/git/GitOperationService.go @@ -27,7 +27,7 @@ type GitOperationService interface { chartProxyReq *bean.ChartProxyReqDto) (string, *commonBean.ChartGitAttribute, error) GitPull(clonedDir string, repoUrl string, appStoreName string) error - CommitValues(chartGitAttr *util.ChartConfig) (commitHash string, commitTime time.Time, err error) + CommitValues(chartGitAttr *ChartConfig) (commitHash string, commitTime time.Time, err error) CommitAndPushAllChanges(clonedDir, commitMsg, userName, userEmailId string) (commitHash string, err error) CreateRepository(dto *bean2.GitOpsConfigDto, userId int32) (string, bool, error) @@ -39,12 +39,12 @@ type GitOperationService interface { type GitOperationServiceImpl struct { logger *zap.SugaredLogger - gitFactory *util.GitFactory + gitFactory *GitFactory gitOpsConfigReadService config.GitOpsConfigReadService chartTemplateService util.ChartTemplateService } -func NewGitOperationServiceImpl(logger *zap.SugaredLogger, gitFactory *util.GitFactory, +func NewGitOperationServiceImpl(logger *zap.SugaredLogger, gitFactory *GitFactory, gitOpsConfigReadService config.GitOpsConfigReadService, chartTemplateService util.ChartTemplateService) *GitOperationServiceImpl { return &GitOperationServiceImpl{ @@ -326,7 +326,7 @@ func (impl *GitOperationServiceImpl) GitPull(clonedDir string, repoUrl string, a return nil } -func (impl *GitOperationServiceImpl) CommitValues(chartGitAttr *util.ChartConfig) (commitHash string, commitTime time.Time, err error) { +func (impl *GitOperationServiceImpl) CommitValues(chartGitAttr *ChartConfig) (commitHash string, commitTime time.Time, err error) { bitbucketMetadata, err := impl.gitOpsConfigReadService.GetBitbucketMetadata() if err != nil { impl.logger.Errorw("error in getting bitbucket metadata", "err", err) diff --git a/internal/util/GitService.go b/pkg/deployment/gitOps/git/GitService.go similarity index 96% rename from internal/util/GitService.go rename to pkg/deployment/gitOps/git/GitService.go index 882772e523..d80e550a6f 100644 --- a/internal/util/GitService.go +++ b/pkg/deployment/gitOps/git/GitService.go @@ -15,7 +15,7 @@ * */ -package util +package git import ( "context" @@ -88,19 +88,19 @@ func (factory *GitFactory) Reload() error { defer func() { util.TriggerGitOpsMetrics("Reload", "GitService", start, err) }() - logger.Infow("reloading gitops details") + factory.logger.Infow("reloading gitops details") cfg, err := GetGitConfig(factory.gitOpsRepository) if err != nil { return err } - gitService := NewGitServiceImpl(cfg, logger, factory.gitCliUtil) + gitService := NewGitServiceImpl(cfg, factory.logger, factory.gitCliUtil) factory.GitService = gitService - client, err := NewGitOpsClient(cfg, logger, gitService, factory.gitOpsRepository) + client, err := NewGitOpsClient(cfg, factory.logger, gitService, factory.gitOpsRepository) if err != nil { return err } factory.Client = client - logger.Infow(" gitops details reload success") + factory.logger.Infow(" gitops details reload success") return nil } @@ -159,15 +159,15 @@ func (factory *GitFactory) NewClientForValidation(gitOpsConfig *bean2.GitOpsConf BitbucketWorkspaceId: gitOpsConfig.BitBucketWorkspaceId, BitbucketProjectKey: gitOpsConfig.BitBucketProjectKey, } - gitService := NewGitServiceImpl(cfg, logger, factory.gitCliUtil) + gitService := NewGitServiceImpl(cfg, factory.logger, factory.gitCliUtil) //factory.GitService = GitService - client, err := NewGitOpsClient(cfg, logger, gitService, factory.gitOpsRepository) + client, err := NewGitOpsClient(cfg, factory.logger, gitService, factory.gitOpsRepository) if err != nil { return client, gitService, err } //factory.Client = client - logger.Infow("client changed successfully", "cfg", cfg) + factory.logger.Infow("client changed successfully", "cfg", cfg) return client, gitService, nil } diff --git a/internal/util/GitServiceAzure.go b/pkg/deployment/gitOps/git/GitServiceAzure.go similarity index 99% rename from internal/util/GitServiceAzure.go rename to pkg/deployment/gitOps/git/GitServiceAzure.go index d47f4f8be4..1bb7dfbbc7 100644 --- a/internal/util/GitServiceAzure.go +++ b/pkg/deployment/gitOps/git/GitServiceAzure.go @@ -1,4 +1,4 @@ -package util +package git import ( "context" @@ -91,7 +91,7 @@ func (impl GitAzureClient) CreateRepository(config *bean2.GitOpsConfigDto) (url detailedErrorGitOpsConfigActions.StageErrorMap[CreateRepoStage] = err return "", true, detailedErrorGitOpsConfigActions } - logger.Infow("repo created ", "r", operationReference.WebUrl) + impl.logger.Infow("repo created ", "r", operationReference.WebUrl) detailedErrorGitOpsConfigActions.SuccessfulStages = append(detailedErrorGitOpsConfigActions.SuccessfulStages, CreateRepoStage) validated, err := impl.ensureProjectAvailabilityOnHttp(config.GitRepoName) if err != nil { diff --git a/internal/util/GitServiceBitbucket.go b/pkg/deployment/gitOps/git/GitServiceBitbucket.go similarity index 99% rename from internal/util/GitServiceBitbucket.go rename to pkg/deployment/gitOps/git/GitServiceBitbucket.go index 8be8359c0e..ec9296f84f 100644 --- a/internal/util/GitServiceBitbucket.go +++ b/pkg/deployment/gitOps/git/GitServiceBitbucket.go @@ -1,4 +1,4 @@ -package util +package git import ( "fmt" @@ -100,7 +100,7 @@ func (impl GitBitbucketClient) CreateRepository(config *bean2.GitOpsConfigDto) ( return "", true, detailedErrorGitOpsConfigActions } repoUrl = fmt.Sprintf(BITBUCKET_CLONE_BASE_URL+"%s/%s.git", repoOptions.Owner, repoOptions.RepoSlug) - logger.Infow("repo created ", "repoUrl", repoUrl) + impl.logger.Infow("repo created ", "repoUrl", repoUrl) detailedErrorGitOpsConfigActions.SuccessfulStages = append(detailedErrorGitOpsConfigActions.SuccessfulStages, CreateRepoStage) validated, err := impl.ensureProjectAvailabilityOnHttp(repoOptions) diff --git a/internal/util/GitServiceGithub.go b/pkg/deployment/gitOps/git/GitServiceGithub.go similarity index 99% rename from internal/util/GitServiceGithub.go rename to pkg/deployment/gitOps/git/GitServiceGithub.go index 12efedebc3..b7cb50c927 100644 --- a/internal/util/GitServiceGithub.go +++ b/pkg/deployment/gitOps/git/GitServiceGithub.go @@ -1,4 +1,4 @@ -package util +package git import ( "context" @@ -99,7 +99,7 @@ func (impl GitHubClient) CreateRepository(config *bean2.GitOpsConfigDto) (url st detailedErrorGitOpsConfigActions.StageErrorMap[CreateRepoStage] = err return "", true, detailedErrorGitOpsConfigActions } - logger.Infow("github repo created ", "r", r.CloneURL) + impl.logger.Infow("github repo created ", "r", r.CloneURL) detailedErrorGitOpsConfigActions.SuccessfulStages = append(detailedErrorGitOpsConfigActions.SuccessfulStages, CreateRepoStage) validated, err := impl.ensureProjectAvailabilityOnHttp(config) diff --git a/internal/util/GitServiceGitlab.go b/pkg/deployment/gitOps/git/GitServiceGitlab.go similarity index 99% rename from internal/util/GitServiceGitlab.go rename to pkg/deployment/gitOps/git/GitServiceGitlab.go index ad34974412..4f8e210698 100644 --- a/internal/util/GitServiceGitlab.go +++ b/pkg/deployment/gitOps/git/GitServiceGitlab.go @@ -1,4 +1,4 @@ -package util +package git import ( "fmt" diff --git a/pkg/gitops/GitOpsConfigService.go b/pkg/gitops/GitOpsConfigService.go index bda050d404..923b31a65d 100644 --- a/pkg/gitops/GitOpsConfigService.go +++ b/pkg/gitops/GitOpsConfigService.go @@ -24,6 +24,7 @@ import ( util4 "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config/bean" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" "math/rand" "net/http" "net/url" @@ -95,7 +96,7 @@ type GitOpsConfigServiceImpl struct { K8sUtil *util4.K8sServiceImpl aCDAuthConfig *util3.ACDAuthConfig clusterService cluster.ClusterService - gitFactory *util.GitFactory + gitFactory *git.GitFactory argoUserService argo.ArgoUserService clusterServiceCD cluster2.ServiceClient gitOpsConfigReadService config.GitOpsConfigReadService @@ -105,7 +106,7 @@ func NewGitOpsConfigServiceImpl(Logger *zap.SugaredLogger, globalEnvVariables *util2.GlobalEnvVariables, gitOpsRepository repository.GitOpsConfigRepository, K8sUtil *util4.K8sServiceImpl, aCDAuthConfig *util3.ACDAuthConfig, clusterService cluster.ClusterService, - gitFactory *util.GitFactory, argoUserService argo.ArgoUserService, + gitFactory *git.GitFactory, argoUserService argo.ArgoUserService, clusterServiceCD cluster2.ServiceClient, gitOpsConfigReadService config.GitOpsConfigReadService) *GitOpsConfigServiceImpl { return &GitOpsConfigServiceImpl{ randSource: rand.NewSource(time.Now().UnixNano()), @@ -290,7 +291,7 @@ func (impl *GitOpsConfigServiceImpl) CreateGitOpsConfig(ctx context.Context, req } } if strings.ToUpper(request.Provider) == bean.BITBUCKET_PROVIDER { - request.Host = util.BITBUCKET_CLONE_BASE_URL + request.BitBucketWorkspaceId + request.Host = git.BITBUCKET_CLONE_BASE_URL + request.BitBucketWorkspaceId } operationComplete := false retryCount := 0 @@ -491,7 +492,7 @@ func (impl *GitOpsConfigServiceImpl) UpdateGitOpsConfig(request *bean2.GitOpsCon } } if strings.ToUpper(request.Provider) == bean.BITBUCKET_PROVIDER { - request.Host = util.BITBUCKET_CLONE_BASE_URL + request.BitBucketWorkspaceId + request.Host = git.BITBUCKET_CLONE_BASE_URL + request.BitBucketWorkspaceId } operationComplete := false retryCount := 0 @@ -679,13 +680,13 @@ func (impl *GitOpsConfigServiceImpl) GitOpsValidateDryRun(config *bean2.GitOpsCo } config.Token = model.Token } - detailedErrorGitOpsConfigActions := util.DetailedErrorGitOpsConfigActions{} + detailedErrorGitOpsConfigActions := git.DetailedErrorGitOpsConfigActions{} detailedErrorGitOpsConfigActions.StageErrorMap = make(map[string]error) /*if strings.ToUpper(config.Provider) == GITHUB_PROVIDER { config.Host = GITHUB_HOST }*/ if strings.ToUpper(config.Provider) == bean.BITBUCKET_PROVIDER { - config.Host = util.BITBUCKET_CLONE_BASE_URL + config.Host = git.BITBUCKET_CLONE_BASE_URL config.BitBucketProjectKey = strings.ToUpper(config.BitBucketProjectKey) } client, gitService, err := impl.gitFactory.NewClientForValidation(config) @@ -793,7 +794,7 @@ func (impl *GitOpsConfigServiceImpl) extractErrorMessageByProvider(err error, pr return err } -func (impl *GitOpsConfigServiceImpl) convertDetailedErrorToResponse(detailedErrorGitOpsConfigActions util.DetailedErrorGitOpsConfigActions) (detailedErrorResponse DetailedErrorGitOpsConfigResponse) { +func (impl *GitOpsConfigServiceImpl) convertDetailedErrorToResponse(detailedErrorGitOpsConfigActions git.DetailedErrorGitOpsConfigActions) (detailedErrorResponse DetailedErrorGitOpsConfigResponse) { detailedErrorResponse.StageErrorMap = make(map[string]string) detailedErrorResponse.SuccessfulStages = detailedErrorGitOpsConfigActions.SuccessfulStages for stage, err := range detailedErrorGitOpsConfigActions.StageErrorMap { diff --git a/wire_gen.go b/wire_gen.go index 4a8ed0becf..09d7f89c6e 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -372,8 +372,8 @@ func InitializeApp() (*App, error) { } chartRefServiceImpl := chartRef.NewChartRefServiceImpl(sugaredLogger, chartRefRepositoryImpl, chartTemplateServiceImpl, utilMergeUtil) gitOpsConfigReadServiceImpl := config.NewGitOpsConfigReadServiceImpl(sugaredLogger, gitOpsConfigRepositoryImpl, userServiceImpl, globalEnvVariables) - gitCliUtil := util.NewGitCliUtil(sugaredLogger) - gitFactory, err := util.NewGitFactory(sugaredLogger, gitOpsConfigRepositoryImpl, gitCliUtil) + gitCliUtil := git.NewGitCliUtil(sugaredLogger) + gitFactory, err := git.NewGitFactory(sugaredLogger, gitOpsConfigRepositoryImpl, gitCliUtil) if err != nil { return nil, err } From c1ab97005cfc64beef909858126fde52740317e8 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 23 Jan 2024 14:09:14 +0530 Subject: [PATCH 35/64] gitops repository usages refactor --- .../telemetry/TelemetryEventClientExtended.go | 19 +++---- cmd/external-app/wire_gen.go | 2 +- .../sql/repository/GitOpsConfigRepository.go | 13 ----- .../common/AppStoreDeploymentCommonService.go | 2 +- .../service/AppStoreDeploymentService.go | 17 ++---- .../deployment/service/InstalledAppService.go | 12 ++--- .../service/InstalledAppService_test.go | 8 --- pkg/cluster/ClusterServiceExtended.go | 24 ++++----- .../gitOps/config/GitOpsConfigReadService.go | 12 +++++ .../gitOps/git/GitOperationService.go | 44 +++++++-------- pkg/deployment/gitOps/git/GitService.go | 54 +------------------ pkg/gitops/GitOpsConfigService.go | 2 +- util/argo/ArgoUserService.go | 24 ++++----- wire_gen.go | 46 ++++++++-------- 14 files changed, 98 insertions(+), 181 deletions(-) diff --git a/client/telemetry/TelemetryEventClientExtended.go b/client/telemetry/TelemetryEventClientExtended.go index cdf27e02b8..59bde30c0a 100644 --- a/client/telemetry/TelemetryEventClientExtended.go +++ b/client/telemetry/TelemetryEventClientExtended.go @@ -3,6 +3,7 @@ package telemetry import ( "encoding/json" cloudProviderIdentifier "github.com/devtron-labs/common-lib/cloud-provider-identifier" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" "net/http" "time" @@ -35,7 +36,6 @@ type TelemetryEventClientImplExtended struct { appListingRepository repository.AppListingRepository ciPipelineRepository pipelineConfig.CiPipelineRepository pipelineRepository pipelineConfig.PipelineRepository - gitOpsConfigRepository repository.GitOpsConfigRepository gitProviderRepository repository.GitProviderRepository dockerArtifactStoreRepository dockerRegistryRepository.DockerArtifactStoreRepository appRepository app.AppRepository @@ -45,6 +45,7 @@ type TelemetryEventClientImplExtended struct { ciTemplateRepository pipelineConfig.CiTemplateRepository chartRepository chartRepoRepository.ChartRepository ciBuildConfigService pipeline.CiBuildConfigService + gitOpsConfigReadService config.GitOpsConfigReadService *TelemetryEventClientImpl } @@ -53,15 +54,16 @@ func NewTelemetryEventClientImplExtended(logger *zap.SugaredLogger, client *http environmentService cluster.EnvironmentService, userService user2.UserService, appListingRepository repository.AppListingRepository, PosthogClient *PosthogClient, ciPipelineRepository pipelineConfig.CiPipelineRepository, pipelineRepository pipelineConfig.PipelineRepository, - gitOpsConfigRepository repository.GitOpsConfigRepository, gitProviderRepository repository.GitProviderRepository, - attributeRepo repository.AttributesRepository, ssoLoginService sso.SSOLoginService, appRepository app.AppRepository, + gitProviderRepository repository.GitProviderRepository, attributeRepo repository.AttributesRepository, + ssoLoginService sso.SSOLoginService, appRepository app.AppRepository, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, dockerArtifactStoreRepository dockerRegistryRepository.DockerArtifactStoreRepository, materialRepository pipelineConfig.MaterialRepository, ciTemplateRepository pipelineConfig.CiTemplateRepository, chartRepository chartRepoRepository.ChartRepository, userAuditService user2.UserAuditService, ciBuildConfigService pipeline.CiBuildConfigService, moduleRepository moduleRepo.ModuleRepository, serverDataStore *serverDataStore.ServerDataStore, helmAppClient client.HelmAppClient, InstalledAppRepository repository2.InstalledAppRepository, userAttributesRepository repository.UserAttributesRepository, - cloudProviderIdentifierService cloudProviderIdentifier.ProviderIdentifierService) (*TelemetryEventClientImplExtended, error) { + cloudProviderIdentifierService cloudProviderIdentifier.ProviderIdentifierService, + gitOpsConfigReadService config.GitOpsConfigReadService) (*TelemetryEventClientImplExtended, error) { cron := cron.New( cron.WithChain()) @@ -71,7 +73,6 @@ func NewTelemetryEventClientImplExtended(logger *zap.SugaredLogger, client *http appListingRepository: appListingRepository, ciPipelineRepository: ciPipelineRepository, pipelineRepository: pipelineRepository, - gitOpsConfigRepository: gitOpsConfigRepository, gitProviderRepository: gitProviderRepository, dockerArtifactStoreRepository: dockerArtifactStoreRepository, appRepository: appRepository, @@ -81,7 +82,7 @@ func NewTelemetryEventClientImplExtended(logger *zap.SugaredLogger, client *http ciTemplateRepository: ciTemplateRepository, chartRepository: chartRepository, ciBuildConfigService: ciBuildConfigService, - + gitOpsConfigReadService: gitOpsConfigReadService, TelemetryEventClientImpl: &TelemetryEventClientImpl{ cron: cron, logger: logger, @@ -233,8 +234,8 @@ func (impl *TelemetryEventClientImplExtended) SendSummaryEvent(eventType string) return err } - gitOps, err := impl.gitOpsConfigRepository.GetAllGitOpsConfig() - if err != nil && err != pg.ErrNoRows { + gitOpsCount, err := impl.gitOpsConfigReadService.GetConfiguredGitOpsCount() + if err != nil { impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) return err } @@ -308,7 +309,7 @@ func (impl *TelemetryEventClientImplExtended) SendSummaryEvent(eventType string) payload.CdCountPerDay = len(cdPipeline) payload.GitAccountsCount = len(gitAccounts) - payload.GitOpsCount = len(gitOps) + payload.GitOpsCount = gitOpsCount payload.HostURL = hostURL payload.DevtronGitVersion = devtronVersion.GitCommit payload.Build = build diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index dd3932c80d..7fd5f2b637 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -286,7 +286,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - appStoreDeploymentServiceImpl := service3.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, clusterInstalledAppsRepositoryImpl, appRepositoryImpl, appStoreDeploymentHelmServiceImpl, appStoreDeploymentHelmServiceImpl, environmentServiceImpl, clusterServiceImpl, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, installedAppVersionHistoryRepositoryImpl, gitOpsConfigRepositoryImpl, deploymentServiceTypeConfig, acdConfig, gitOpsConfigReadServiceImpl, gitOperationServiceImpl) + appStoreDeploymentServiceImpl := service3.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, clusterInstalledAppsRepositoryImpl, appRepositoryImpl, appStoreDeploymentHelmServiceImpl, appStoreDeploymentHelmServiceImpl, environmentServiceImpl, clusterServiceImpl, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, installedAppVersionHistoryRepositoryImpl, deploymentServiceTypeConfig, acdConfig, gitOpsConfigReadServiceImpl, gitOperationServiceImpl) appStoreDeploymentRestHandlerImpl := appStoreDeployment.NewAppStoreDeploymentRestHandlerImpl(sugaredLogger, userServiceImpl, enforcerImpl, enforcerUtilImpl, enforcerUtilHelmImpl, appStoreDeploymentServiceImpl, validate, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, helmUserServiceImpl, attributesServiceImpl) appStoreDeploymentRouterImpl := appStoreDeployment.NewAppStoreDeploymentRouterImpl(appStoreDeploymentRestHandlerImpl) chartProviderServiceImpl := chartProvider.NewChartProviderServiceImpl(sugaredLogger, chartRepoRepositoryImpl, chartRepositoryServiceImpl, dockerArtifactStoreRepositoryImpl, ociRegistryConfigRepositoryImpl) diff --git a/internal/sql/repository/GitOpsConfigRepository.go b/internal/sql/repository/GitOpsConfigRepository.go index 557cfaf488..8e682f5bbf 100644 --- a/internal/sql/repository/GitOpsConfigRepository.go +++ b/internal/sql/repository/GitOpsConfigRepository.go @@ -32,7 +32,6 @@ type GitOpsConfigRepository interface { GetGitOpsConfigActive() (*GitOpsConfig, error) GetConnection() *pg.DB GetEmailIdFromActiveGitOpsConfig() (string, error) - IsGitOpsConfigured() (bool, error) } type GitOpsConfigRepositoryImpl struct { @@ -109,15 +108,3 @@ func (impl *GitOpsConfigRepositoryImpl) GetEmailIdFromActiveGitOpsConfig() (stri Where("active = ?", true).Select(&emailId) return emailId, err } - -func (impl *GitOpsConfigRepositoryImpl) IsGitOpsConfigured() (bool, error) { - gitOpsConfig, err := impl.GetGitOpsConfigActive() - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("Error while getting git-ops config from DB to check if git-ops configured or not", "err", err) - return false, err - } - if gitOpsConfig != nil && gitOpsConfig.Id > 0 { - return true, nil - } - return false, nil -} diff --git a/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go b/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go index 68146304c0..a62122f071 100644 --- a/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go +++ b/pkg/appStore/deployment/common/AppStoreDeploymentCommonService.go @@ -539,7 +539,7 @@ func (impl AppStoreDeploymentCommonServiceImpl) GitOpsOperations(manifestRespons } space := regexp.MustCompile(`\s+`) appStoreName := space.ReplaceAllString(installAppVersionRequest.AppName, "-") - clonedDir := impl.gitFactory.GitWorkingDir + "" + appStoreName + clonedDir := git.GIT_WORKING_DIR + "" + appStoreName // Checking this is the first time chart has been pushed , if yes requirements.yaml has been already pushed with chart as there was sync-delay with github api. // step-2 commit dependencies and values in git diff --git a/pkg/appStore/deployment/service/AppStoreDeploymentService.go b/pkg/appStore/deployment/service/AppStoreDeploymentService.go index c8f4db9f2e..203933a51d 100644 --- a/pkg/appStore/deployment/service/AppStoreDeploymentService.go +++ b/pkg/appStore/deployment/service/AppStoreDeploymentService.go @@ -28,7 +28,6 @@ import ( openapi2 "github.com/devtron-labs/devtron/api/openapi/openapiClient" "github.com/devtron-labs/devtron/client/argocdServer" "github.com/devtron-labs/devtron/internal/constants" - repository2 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/helper" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" @@ -104,7 +103,6 @@ type AppStoreDeploymentServiceImpl struct { helmAppService client.HelmAppService appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService installedAppRepositoryHistory repository.InstalledAppVersionHistoryRepository - gitOpsRepository repository2.GitOpsConfigRepository deploymentTypeConfig *DeploymentServiceTypeConfig aCDConfig *argocdServer.ACDConfig gitOpsConfigReadService config.GitOpsConfigReadService @@ -117,7 +115,7 @@ func NewAppStoreDeploymentServiceImpl(logger *zap.SugaredLogger, installedAppRep appStoreDeploymentHelmService appStoreDeploymentTool.AppStoreDeploymentHelmService, appStoreDeploymentArgoCdService appStoreDeploymentTool.AppStoreDeploymentArgoCdService, environmentService cluster.EnvironmentService, clusterService cluster.ClusterService, helmAppService client.HelmAppService, appStoreDeploymentCommonService appStoreDeploymentCommon.AppStoreDeploymentCommonService, - installedAppRepositoryHistory repository.InstalledAppVersionHistoryRepository, gitOpsRepository repository2.GitOpsConfigRepository, + installedAppRepositoryHistory repository.InstalledAppVersionHistoryRepository, deploymentTypeConfig *DeploymentServiceTypeConfig, aCDConfig *argocdServer.ACDConfig, gitOpsConfigReadService config.GitOpsConfigReadService, gitOperationService git.GitOperationService) *AppStoreDeploymentServiceImpl { @@ -137,7 +135,6 @@ func NewAppStoreDeploymentServiceImpl(logger *zap.SugaredLogger, installedAppRep helmAppService: helmAppService, appStoreDeploymentCommonService: appStoreDeploymentCommonService, installedAppRepositoryHistory: installedAppRepositoryHistory, - gitOpsRepository: gitOpsRepository, deploymentTypeConfig: deploymentTypeConfig, aCDConfig: aCDConfig, gitOpsConfigReadService: gitOpsConfigReadService, @@ -149,17 +146,11 @@ func NewAppStoreDeploymentServiceImpl(logger *zap.SugaredLogger, installedAppRep func (impl AppStoreDeploymentServiceImpl) AppStoreDeployOperationDB(installAppVersionRequest *appStoreBean.InstallAppVersionDTO, tx *pg.Tx, skipAppCreation bool) (*appStoreBean.InstallAppVersionDTO, error) { var isInternalUse = impl.deploymentTypeConfig.IsInternalUse - - isGitOpsConfigured := false - gitOpsConfig, err := impl.gitOpsRepository.GetGitOpsConfigActive() - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("GetGitOpsConfigActive, error while getting", "err", err) + isGitOpsConfigured, err := impl.gitOpsConfigReadService.IsGitOpsConfigured() + if err != nil { + impl.logger.Errorw("error while checking if gitOps is configured", "err", err) return nil, err } - if gitOpsConfig != nil && gitOpsConfig.Id > 0 { - isGitOpsConfigured = true - } - if isInternalUse && !isGitOpsConfigured && installAppVersionRequest.DeploymentAppType == util.PIPELINE_DEPLOYMENT_TYPE_ACD { impl.logger.Errorw("gitops not configured but selected for CD") err := &util.ApiError{ diff --git a/pkg/appStore/deployment/service/InstalledAppService.go b/pkg/appStore/deployment/service/InstalledAppService.go index 5566598f01..438e8adbf1 100644 --- a/pkg/appStore/deployment/service/InstalledAppService.go +++ b/pkg/appStore/deployment/service/InstalledAppService.go @@ -69,7 +69,6 @@ import ( pubsub "github.com/devtron-labs/common-lib/pubsub-lib" bean2 "github.com/devtron-labs/devtron/api/bean" application2 "github.com/devtron-labs/devtron/client/argocdServer/application" - repository3 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/bean" cluster2 "github.com/devtron-labs/devtron/pkg/cluster" @@ -108,7 +107,6 @@ type InstalledAppServiceImpl struct { chartGroupDeploymentRepository repository6.ChartGroupDeploymentRepository envService cluster2.EnvironmentService aCDAuthConfig *util2.ACDAuthConfig - gitOpsRepository repository3.GitOpsConfigRepository userService user.UserService appStoreDeploymentService AppStoreDeploymentService appStoreDeploymentFullModeService appStoreDeploymentFullMode.AppStoreDeploymentFullModeService @@ -131,13 +129,10 @@ func NewInstalledAppServiceImpl(logger *zap.SugaredLogger, installedAppRepository repository2.InstalledAppRepository, appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository, environmentRepository repository5.EnvironmentRepository, teamRepository repository4.TeamRepository, - appRepository app.AppRepository, - acdClient application2.ServiceClient, - appStoreValuesService service.AppStoreValuesService, - pubsubClient *pubsub.PubSubClientServiceImpl, + appRepository app.AppRepository, acdClient application2.ServiceClient, + appStoreValuesService service.AppStoreValuesService, pubsubClient *pubsub.PubSubClientServiceImpl, chartGroupDeploymentRepository repository6.ChartGroupDeploymentRepository, - envService cluster2.EnvironmentService, - aCDAuthConfig *util2.ACDAuthConfig, gitOpsRepository repository3.GitOpsConfigRepository, userService user.UserService, + envService cluster2.EnvironmentService, aCDAuthConfig *util2.ACDAuthConfig, userService user.UserService, appStoreDeploymentFullModeService appStoreDeploymentFullMode.AppStoreDeploymentFullModeService, appStoreDeploymentService AppStoreDeploymentService, installedAppRepositoryHistory repository2.InstalledAppVersionHistoryRepository, @@ -161,7 +156,6 @@ func NewInstalledAppServiceImpl(logger *zap.SugaredLogger, chartGroupDeploymentRepository: chartGroupDeploymentRepository, envService: envService, aCDAuthConfig: aCDAuthConfig, - gitOpsRepository: gitOpsRepository, userService: userService, appStoreDeploymentService: appStoreDeploymentService, appStoreDeploymentFullModeService: appStoreDeploymentFullModeService, diff --git a/pkg/appStore/deployment/service/InstalledAppService_test.go b/pkg/appStore/deployment/service/InstalledAppService_test.go index 2c2f90e7be..7bf2ca08c3 100644 --- a/pkg/appStore/deployment/service/InstalledAppService_test.go +++ b/pkg/appStore/deployment/service/InstalledAppService_test.go @@ -8,7 +8,6 @@ import ( "github.com/devtron-labs/devtron/client/argocdServer" "github.com/devtron-labs/devtron/client/argocdServer/application" repository2 "github.com/devtron-labs/devtron/client/argocdServer/repository" - repository3 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/util" repository5 "github.com/devtron-labs/devtron/pkg/appStore/chartGroup/repository" @@ -43,7 +42,6 @@ func TestInstalledAppServiceImpl_DeployDefaultChartOnCluster(t *testing.T) { ArgoK8sClient argocdServer.ArgoK8sClient gitFactory *git.GitFactory aCDAuthConfig *util2.ACDAuthConfig - gitOpsRepository repository3.GitOpsConfigRepository userService user.UserService appStoreDeploymentService AppStoreDeploymentService appStoreDeploymentFullModeService appStoreDeploymentFullMode.AppStoreDeploymentFullModeService @@ -66,8 +64,6 @@ func TestInstalledAppServiceImpl_DeployDefaultChartOnCluster(t *testing.T) { impl := &InstalledAppServiceImpl{ logger: tt.fields.logger, installedAppRepository: tt.fields.installedAppRepository, - chartTemplateService: tt.fields.chartTemplateService, - repositoryService: tt.fields.repositoryService, appStoreApplicationVersionRepository: tt.fields.appStoreApplicationVersionRepository, environmentRepository: tt.fields.environmentRepository, teamRepository: tt.fields.teamRepository, @@ -75,13 +71,9 @@ func TestInstalledAppServiceImpl_DeployDefaultChartOnCluster(t *testing.T) { acdClient: tt.fields.acdClient, appStoreValuesService: tt.fields.appStoreValuesService, pubsubClient: tt.fields.pubsubClient, - tokenCache: tt.fields.tokenCache, chartGroupDeploymentRepository: tt.fields.chartGroupDeploymentRepository, envService: tt.fields.envService, - ArgoK8sClient: tt.fields.ArgoK8sClient, - gitFactory: tt.fields.gitFactory, aCDAuthConfig: tt.fields.aCDAuthConfig, - gitOpsRepository: tt.fields.gitOpsRepository, userService: tt.fields.userService, appStoreDeploymentService: tt.fields.appStoreDeploymentService, appStoreDeploymentFullModeService: tt.fields.appStoreDeploymentFullModeService, diff --git a/pkg/cluster/ClusterServiceExtended.go b/pkg/cluster/ClusterServiceExtended.go index 5a11aee3a0..e18865e2b9 100644 --- a/pkg/cluster/ClusterServiceExtended.go +++ b/pkg/cluster/ClusterServiceExtended.go @@ -3,6 +3,7 @@ package cluster import ( "context" "fmt" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" "net/http" "strings" "time" @@ -10,7 +11,6 @@ import ( cluster3 "github.com/argoproj/argo-cd/v2/pkg/apiclient/cluster" "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" "github.com/devtron-labs/common-lib/utils/k8s" - repository3 "github.com/devtron-labs/devtron/internal/sql/repository" repository5 "github.com/devtron-labs/devtron/pkg/auth/user/repository" "github.com/devtron-labs/devtron/pkg/k8s/informer" @@ -26,12 +26,12 @@ import ( // extends ClusterServiceImpl and enhances method of ClusterService with full mode specific errors type ClusterServiceImplExtended struct { - environmentRepository repository.EnvironmentRepository - grafanaClient grafana.GrafanaClient - installedAppRepository repository2.InstalledAppRepository - clusterServiceCD cluster2.ServiceClient - K8sInformerFactory informer.K8sInformerFactory - gitOpsRepository repository3.GitOpsConfigRepository + environmentRepository repository.EnvironmentRepository + grafanaClient grafana.GrafanaClient + installedAppRepository repository2.InstalledAppRepository + clusterServiceCD cluster2.ServiceClient + K8sInformerFactory informer.K8sInformerFactory + gitOpsConfigReadService config.GitOpsConfigReadService *ClusterServiceImpl } @@ -39,14 +39,14 @@ func NewClusterServiceImplExtended(repository repository.ClusterRepository, envi grafanaClient grafana.GrafanaClient, logger *zap.SugaredLogger, installedAppRepository repository2.InstalledAppRepository, K8sUtil *k8s.K8sServiceImpl, clusterServiceCD cluster2.ServiceClient, K8sInformerFactory informer.K8sInformerFactory, - gitOpsRepository repository3.GitOpsConfigRepository, userAuthRepository repository5.UserAuthRepository, - userRepository repository5.UserRepository, roleGroupRepository repository5.RoleGroupRepository) *ClusterServiceImplExtended { + userAuthRepository repository5.UserAuthRepository, + userRepository repository5.UserRepository, roleGroupRepository repository5.RoleGroupRepository, + gitOpsConfigReadService config.GitOpsConfigReadService) *ClusterServiceImplExtended { clusterServiceExt := &ClusterServiceImplExtended{ environmentRepository: environmentRepository, grafanaClient: grafanaClient, installedAppRepository: installedAppRepository, clusterServiceCD: clusterServiceCD, - gitOpsRepository: gitOpsRepository, ClusterServiceImpl: &ClusterServiceImpl{ clusterRepository: repository, logger: logger, @@ -157,7 +157,7 @@ func (impl *ClusterServiceImplExtended) FindAllExceptVirtual() ([]*ClusterBean, } func (impl *ClusterServiceImplExtended) Update(ctx context.Context, bean *ClusterBean, userId int32) (*ClusterBean, error) { - isGitOpsConfigured, err1 := impl.gitOpsRepository.IsGitOpsConfigured() + isGitOpsConfigured, err1 := impl.gitOpsConfigReadService.IsGitOpsConfigured() if err1 != nil { return nil, err1 } @@ -337,7 +337,7 @@ func (impl *ClusterServiceImplExtended) CreateGrafanaDataSource(clusterBean *Clu } func (impl *ClusterServiceImplExtended) Save(ctx context.Context, bean *ClusterBean, userId int32) (*ClusterBean, error) { - isGitOpsConfigured, err := impl.gitOpsRepository.IsGitOpsConfigured() + isGitOpsConfigured, err := impl.gitOpsConfigReadService.IsGitOpsConfigured() if err != nil { return nil, err } diff --git a/pkg/deployment/gitOps/config/GitOpsConfigReadService.go b/pkg/deployment/gitOps/config/GitOpsConfigReadService.go index 6daf96c6d3..851cb22c48 100644 --- a/pkg/deployment/gitOps/config/GitOpsConfigReadService.go +++ b/pkg/deployment/gitOps/config/GitOpsConfigReadService.go @@ -20,6 +20,7 @@ type GitOpsConfigReadService interface { GetGitOpsRepoNameFromUrl(gitRepoUrl string) string GetBitbucketMetadata() (*bean.BitbucketProviderMetadata, error) GetGitOpsConfigActive() (*bean2.GitOpsConfigDto, error) + GetConfiguredGitOpsCount() (int, error) } type GitOpsConfigReadServiceImpl struct { @@ -131,3 +132,14 @@ func (impl *GitOpsConfigReadServiceImpl) GetGitOpsConfigActive() (*bean2.GitOpsC } return config, err } + +func (impl *GitOpsConfigReadServiceImpl) GetConfiguredGitOpsCount() (int, error) { + count := 0 + models, err := impl.gitOpsRepository.GetAllGitOpsConfig() + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error, GetGitOpsConfigActive", "err", err) + return count, err + } + count = len(models) + return count, nil +} diff --git a/pkg/deployment/gitOps/git/GitOperationService.go b/pkg/deployment/gitOps/git/GitOperationService.go index 6fa5f879d4..0f9c9e3a18 100644 --- a/pkg/deployment/gitOps/git/GitOperationService.go +++ b/pkg/deployment/gitOps/git/GitOperationService.go @@ -90,21 +90,16 @@ func (impl *GitOperationServiceImpl) CreateGitRepositoryForApp(gitOpsRepoName, b func (impl *GitOperationServiceImpl) PushChartToGitRepo(gitOpsRepoName, referenceTemplate, version, tempReferenceTemplateDir string, repoUrl string, userId int32) (err error) { chartDir := fmt.Sprintf("%s-%s", gitOpsRepoName, impl.chartTemplateService.GetDir()) - clonedDir := impl.gitFactory.GitService.GetCloneDirectory(chartDir) - if _, err := os.Stat(clonedDir); os.IsNotExist(err) { - clonedDir, err = impl.gitFactory.GitService.Clone(repoUrl, chartDir) - if err != nil { - impl.logger.Errorw("error in cloning repo", "url", repoUrl, "err", err) - return err - } - } else { - err = impl.GitPull(clonedDir, repoUrl, gitOpsRepoName) - if err != nil { - impl.logger.Errorw("error in pulling git repo", "url", repoUrl, "err", err) - return err - } + clonedDir, err := impl.GetClonedDir(chartDir, repoUrl) + if err != nil { + impl.logger.Errorw("error in cloning repo", "url", repoUrl, "err", err) + return err + } + err = impl.GitPull(clonedDir, repoUrl, gitOpsRepoName) + if err != nil { + impl.logger.Errorw("error in pulling git repo", "url", repoUrl, "err", err) + return err } - dir := filepath.Join(clonedDir, referenceTemplate, version) pushChartToGit := true @@ -262,18 +257,15 @@ func (impl *GitOperationServiceImpl) createAndPushToGitChartProxy(appStoreName, } chartDir := fmt.Sprintf("%s-%s", chartProxyReq.AppName, impl.chartTemplateService.GetDir()) - clonedDir := impl.gitFactory.GitService.GetCloneDirectory(chartDir) - if _, err := os.Stat(clonedDir); os.IsNotExist(err) { - clonedDir, err = impl.gitFactory.GitService.Clone(repoUrl, chartDir) - if err != nil { - impl.logger.Errorw("error in cloning repo", "url", repoUrl, "err", err) - return nil, err - } - } else { - err = impl.GitPull(clonedDir, repoUrl, appStoreName) - if err != nil { - return nil, err - } + clonedDir, err := impl.GetClonedDir(chartDir, repoUrl) + if err != nil { + impl.logger.Errorw("error in cloning repo", "url", repoUrl, "err", err) + return nil, err + } + + err = impl.GitPull(clonedDir, repoUrl, appStoreName) + if err != nil { + return nil, err } acdAppName := fmt.Sprintf("%s-%s", chartProxyReq.AppName, envName) diff --git a/pkg/deployment/gitOps/git/GitService.go b/pkg/deployment/gitOps/git/GitService.go index d80e550a6f..6f0f3815d1 100644 --- a/pkg/deployment/gitOps/git/GitService.go +++ b/pkg/deployment/gitOps/git/GitService.go @@ -21,7 +21,6 @@ import ( "context" "fmt" "github.com/devtron-labs/devtron/util" - "io/ioutil" "net/url" "path/filepath" "time" @@ -63,7 +62,6 @@ type GitClient interface { type GitFactory struct { Client GitClient GitService GitService - GitWorkingDir string logger *zap.SugaredLogger gitOpsRepository repository.GitOpsConfigRepository gitCliUtil *GitCliUtil @@ -150,7 +148,6 @@ func (factory *GitFactory) NewClientForValidation(gitOpsConfig *bean2.GitOpsConf GitlabGroupId: gitOpsConfig.GitLabGroupId, GitToken: gitOpsConfig.Token, GitUserName: gitOpsConfig.Username, - GitWorkingDir: GIT_WORKING_DIR, GithubOrganization: gitOpsConfig.GitHubOrgId, GitProvider: gitOpsConfig.Provider, GitHost: gitOpsConfig.Host, @@ -186,7 +183,6 @@ func NewGitFactory(logger *zap.SugaredLogger, gitOpsRepository repository.GitOps logger: logger, GitService: gitService, gitOpsRepository: gitOpsRepository, - GitWorkingDir: cfg.GitWorkingDir, gitCliUtil: gitCliUtil, }, nil } @@ -196,7 +192,6 @@ type GitConfig struct { GitlabGroupPath string //local GitToken string //not null // public GitUserName string //not null // public - GitWorkingDir string //working directory for git. might use pvc GithubOrganization string GitProvider string // SUPPORTED VALUES GITHUB, GITLAB GitHost string @@ -225,7 +220,6 @@ func GetGitConfig(gitOpsRepository repository.GitOpsConfigRepository) (*GitConfi GitlabGroupId: gitOpsConfig.GitLabGroupId, GitToken: gitOpsConfig.Token, GitUserName: gitOpsConfig.Username, - GitWorkingDir: GIT_WORKING_DIR, GithubOrganization: gitOpsConfig.GitHubOrgId, GitProvider: gitOpsConfig.Provider, GitHost: gitOpsConfig.Host, @@ -271,15 +265,12 @@ type ChartConfig struct { type GitService interface { Clone(url, targetDir string) (clonedDir string, err error) CommitAndPushAllChanges(repoRoot, commitMsg, name, emailId string) (commitHash string, err error) - ForceResetHead(repoRoot string) (err error) - CommitValues(config *ChartConfig) (commitHash string, err error) GetCloneDirectory(targetDir string) (clonedDir string) Pull(repoRoot string) (err error) } type GitServiceImpl struct { Auth *http.BasicAuth - config *GitConfig logger *zap.SugaredLogger gitCliUtil *GitCliUtil } @@ -289,18 +280,16 @@ func NewGitServiceImpl(config *GitConfig, logger *zap.SugaredLogger, GitCliUtil return &GitServiceImpl{ Auth: auth, logger: logger, - config: config, gitCliUtil: GitCliUtil, } } func (impl GitServiceImpl) GetCloneDirectory(targetDir string) (clonedDir string) { - start := time.Now() defer func() { util.TriggerGitOpsMetrics("GetCloneDirectory", "GitService", start, nil) }() - clonedDir = filepath.Join(impl.config.GitWorkingDir, targetDir) + clonedDir = filepath.Join(GIT_WORKING_DIR, targetDir) return clonedDir } @@ -310,7 +299,7 @@ func (impl GitServiceImpl) Clone(url, targetDir string) (clonedDir string, err e util.TriggerGitOpsMetrics("Clone", "GitService", start, err) }() impl.logger.Debugw("git checkout ", "url", url, "dir", targetDir) - clonedDir = filepath.Join(impl.config.GitWorkingDir, targetDir) + clonedDir = filepath.Join(GIT_WORKING_DIR, targetDir) _, errorMsg, err := impl.gitCliUtil.Clone(clonedDir, url, impl.Auth.Username, impl.Auth.Password) if err != nil { impl.logger.Errorw("error in git checkout", "url", url, "targetDir", targetDir, "err", err) @@ -373,45 +362,6 @@ func (impl GitServiceImpl) getRepoAndWorktree(repoRoot string) (*git.Repository, return r, w, err } -func (impl GitServiceImpl) ForceResetHead(repoRoot string) (err error) { - start := time.Now() - defer func() { - util.TriggerGitOpsMetrics("ForceResetHead", "GitService", start, err) - }() - _, workTree, err := impl.getRepoAndWorktree(repoRoot) - if err != nil { - return err - } - err = workTree.Reset(&git.ResetOptions{Mode: git.HardReset}) - if err != nil { - return err - } - err = workTree.Pull(&git.PullOptions{ - Auth: impl.Auth, - Force: true, - SingleBranch: true, - }) - return err -} - -func (impl GitServiceImpl) CommitValues(config *ChartConfig) (commitHash string, err error) { - //TODO acquire lock - start := time.Now() - defer func() { - util.TriggerGitOpsMetrics("CommitValues", "GitService", start, err) - }() - gitDir := filepath.Join(impl.config.GitWorkingDir, config.ChartName) - if err != nil { - return "", err - } - err = ioutil.WriteFile(filepath.Join(gitDir, config.ChartLocation, config.FileName), []byte(config.FileContent), 0600) - if err != nil { - return "", err - } - hash, err := impl.CommitAndPushAllChanges(gitDir, config.ReleaseMessage, "devtron bot", "devtron-bot@devtron.ai") - return hash, err -} - func (impl GitServiceImpl) Pull(repoRoot string) (err error) { start := time.Now() defer func() { diff --git a/pkg/gitops/GitOpsConfigService.go b/pkg/gitops/GitOpsConfigService.go index 923b31a65d..06e8f7052e 100644 --- a/pkg/gitops/GitOpsConfigService.go +++ b/pkg/gitops/GitOpsConfigService.go @@ -322,7 +322,7 @@ func (impl *GitOpsConfigServiceImpl) CreateGitOpsConfig(ctx context.Context, req } // if git-ops config is created/saved successfully (just before transaction commit) and this was first git-ops config, then upsert clusters in acd - isGitOpsConfigured, err := impl.gitOpsRepository.IsGitOpsConfigured() + isGitOpsConfigured, err := impl.gitOpsConfigReadService.IsGitOpsConfigured() if err != nil { return nil, err } diff --git a/util/argo/ArgoUserService.go b/util/argo/ArgoUserService.go index 80af547c6c..75715dd704 100644 --- a/util/argo/ArgoUserService.go +++ b/util/argo/ArgoUserService.go @@ -9,10 +9,9 @@ import ( "github.com/devtron-labs/devtron/client/argocdServer" "github.com/devtron-labs/devtron/client/argocdServer/connection" "github.com/devtron-labs/devtron/client/argocdServer/session" - "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/pkg/cluster" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" util2 "github.com/devtron-labs/devtron/util" - "github.com/go-pg/pg" "go.uber.org/zap" "golang.org/x/crypto/bcrypt" apiv1 "k8s.io/api/core/v1" @@ -48,22 +47,25 @@ type ArgoUserServiceImpl struct { clusterService cluster.ClusterService devtronSecretConfig *util2.DevtronSecretConfig runTimeConfig *client.RuntimeConfig - gitOpsRepository repository.GitOpsConfigRepository argoCDConnectionManager connection.ArgoCDConnectionManager versionService argocdServer.VersionService k8sUtil *k8s.K8sServiceImpl + gitOpsConfigReadService config.GitOpsConfigReadService } -func NewArgoUserServiceImpl(Logger *zap.SugaredLogger, clusterService cluster.ClusterService, devtronSecretConfig *util2.DevtronSecretConfig, runTimeConfig *client.RuntimeConfig, gitOpsRepository repository.GitOpsConfigRepository, argoCDConnectionManager connection.ArgoCDConnectionManager, versionService argocdServer.VersionService, k8sUtil *k8s.K8sServiceImpl) (*ArgoUserServiceImpl, error) { +func NewArgoUserServiceImpl(Logger *zap.SugaredLogger, clusterService cluster.ClusterService, + devtronSecretConfig *util2.DevtronSecretConfig, runTimeConfig *client.RuntimeConfig, + argoCDConnectionManager connection.ArgoCDConnectionManager, versionService argocdServer.VersionService, + k8sUtil *k8s.K8sServiceImpl, gitOpsConfigReadService config.GitOpsConfigReadService) (*ArgoUserServiceImpl, error) { argoUserServiceImpl := &ArgoUserServiceImpl{ logger: Logger, clusterService: clusterService, devtronSecretConfig: devtronSecretConfig, runTimeConfig: runTimeConfig, - gitOpsRepository: gitOpsRepository, argoCDConnectionManager: argoCDConnectionManager, versionService: versionService, k8sUtil: k8sUtil, + gitOpsConfigReadService: gitOpsConfigReadService, } if !runTimeConfig.LocalDevMode { go argoUserServiceImpl.ValidateGitOpsAndGetOrUpdateArgoCdUserDetail() @@ -72,7 +74,7 @@ func NewArgoUserServiceImpl(Logger *zap.SugaredLogger, clusterService cluster.Cl } func (impl *ArgoUserServiceImpl) ValidateGitOpsAndGetOrUpdateArgoCdUserDetail() string { - isGitOpsConfigured, err := impl.gitOpsRepository.IsGitOpsConfigured() + isGitOpsConfigured, err := impl.gitOpsConfigReadService.IsGitOpsConfigured() if err != nil || !isGitOpsConfigured { return "" } @@ -163,15 +165,11 @@ func (impl *ArgoUserServiceImpl) createNewArgoCdTokenForDevtron(username, passwo // note: this function also called for no gitops case, where apps are installed via helm func (impl *ArgoUserServiceImpl) GetLatestDevtronArgoCdUserToken() (string, error) { - isGitOpsConfigured := false - gitOpsConfig, err := impl.gitOpsRepository.GetGitOpsConfigActive() - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("GetGitOpsConfigActive, error while getting", "err", err) + isGitOpsConfigured, err := impl.gitOpsConfigReadService.IsGitOpsConfigured() + if err != nil { + impl.logger.Errorw("error while checking if gitOps is configured", "err", err) return "", err } - if gitOpsConfig != nil && gitOpsConfig.Id > 0 { - isGitOpsConfigured = true - } if !isGitOpsConfigured { //here acd token only required in context for argo cd calls return "", nil diff --git a/wire_gen.go b/wire_gen.go index 09d7f89c6e..20d32eb2d1 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -228,24 +228,12 @@ func InitializeApp() (*App, error) { serviceClientImpl := cluster.NewServiceClientImpl(sugaredLogger, argoCDConnectionManagerImpl) v := informer.NewGlobalMapClusterNamespace() k8sInformerFactoryImpl := informer.NewK8sInformerFactoryImpl(sugaredLogger, v, runtimeConfig, k8sServiceImpl) - gitOpsConfigRepositoryImpl := repository.NewGitOpsConfigRepositoryImpl(sugaredLogger, db) defaultAuthPolicyRepositoryImpl := repository4.NewDefaultAuthPolicyRepositoryImpl(db, sugaredLogger) defaultAuthRoleRepositoryImpl := repository4.NewDefaultAuthRoleRepositoryImpl(db, sugaredLogger) userAuthRepositoryImpl := repository4.NewUserAuthRepositoryImpl(db, sugaredLogger, defaultAuthPolicyRepositoryImpl, defaultAuthRoleRepositoryImpl) userRepositoryImpl := repository4.NewUserRepositoryImpl(db, sugaredLogger) roleGroupRepositoryImpl := repository4.NewRoleGroupRepositoryImpl(db, sugaredLogger) - clusterServiceImplExtended := cluster2.NewClusterServiceImplExtended(clusterRepositoryImpl, environmentRepositoryImpl, grafanaClientImpl, sugaredLogger, installedAppRepositoryImpl, k8sServiceImpl, serviceClientImpl, k8sInformerFactoryImpl, gitOpsConfigRepositoryImpl, userAuthRepositoryImpl, userRepositoryImpl, roleGroupRepositoryImpl) - helmClientConfig, err := client3.GetConfig() - if err != nil { - return nil, err - } - helmAppClientImpl := client3.NewHelmAppClientImpl(sugaredLogger, helmClientConfig) - pumpImpl := connector.NewPumpImpl(sugaredLogger) - teamRepositoryImpl := team.NewTeamRepositoryImpl(db) - appRepositoryImpl := app.NewAppRepositoryImpl(db, sugaredLogger) - enforcerUtilHelmImpl := rbac.NewEnforcerUtilHelmImpl(sugaredLogger, clusterRepositoryImpl, teamRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, installedAppRepositoryImpl) - serverDataStoreServerDataStore := serverDataStore.InitServerDataStore() - appStoreApplicationVersionRepositoryImpl := appStoreDiscoverRepository.NewAppStoreApplicationVersionRepositoryImpl(sugaredLogger, db) + gitOpsConfigRepositoryImpl := repository.NewGitOpsConfigRepositoryImpl(sugaredLogger, db) k8sClient, err := client2.NewK8sClient(runtimeConfig) if err != nil { return nil, err @@ -260,7 +248,6 @@ func InitializeApp() (*App, error) { } apiTokenSecretStore := apiTokenAuth.InitApiTokenSecretStore() sessionManager := middleware.NewSessionManager(settings, dexConfig, apiTokenSecretStore) - loginService := middleware.NewUserLogin(sessionManager, k8sClient) rbacPolicyDataRepositoryImpl := repository4.NewRbacPolicyDataRepositoryImpl(sugaredLogger, db) rbacRoleDataRepositoryImpl := repository4.NewRbacRoleDataRepositoryImpl(sugaredLogger, db) rbacDataCacheFactoryImpl := repository4.NewRbacDataCacheFactoryImpl(sugaredLogger, rbacPolicyDataRepositoryImpl, rbacRoleDataRepositoryImpl) @@ -268,6 +255,24 @@ func InitializeApp() (*App, error) { userAuditRepositoryImpl := repository4.NewUserAuditRepositoryImpl(db) userAuditServiceImpl := user.NewUserAuditServiceImpl(sugaredLogger, userAuditRepositoryImpl) userServiceImpl := user.NewUserServiceImpl(userAuthRepositoryImpl, sugaredLogger, userRepositoryImpl, roleGroupRepositoryImpl, sessionManager, userCommonServiceImpl, userAuditServiceImpl) + globalEnvVariables, err := util2.GetGlobalEnvVariables() + if err != nil { + return nil, err + } + gitOpsConfigReadServiceImpl := config.NewGitOpsConfigReadServiceImpl(sugaredLogger, gitOpsConfigRepositoryImpl, userServiceImpl, globalEnvVariables) + clusterServiceImplExtended := cluster2.NewClusterServiceImplExtended(clusterRepositoryImpl, environmentRepositoryImpl, grafanaClientImpl, sugaredLogger, installedAppRepositoryImpl, k8sServiceImpl, serviceClientImpl, k8sInformerFactoryImpl, userAuthRepositoryImpl, userRepositoryImpl, roleGroupRepositoryImpl, gitOpsConfigReadServiceImpl) + helmClientConfig, err := client3.GetConfig() + if err != nil { + return nil, err + } + helmAppClientImpl := client3.NewHelmAppClientImpl(sugaredLogger, helmClientConfig) + pumpImpl := connector.NewPumpImpl(sugaredLogger) + teamRepositoryImpl := team.NewTeamRepositoryImpl(db) + appRepositoryImpl := app.NewAppRepositoryImpl(db, sugaredLogger) + enforcerUtilHelmImpl := rbac.NewEnforcerUtilHelmImpl(sugaredLogger, clusterRepositoryImpl, teamRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, installedAppRepositoryImpl) + serverDataStoreServerDataStore := serverDataStore.InitServerDataStore() + appStoreApplicationVersionRepositoryImpl := appStoreDiscoverRepository.NewAppStoreApplicationVersionRepositoryImpl(sugaredLogger, db) + loginService := middleware.NewUserLogin(sessionManager, k8sClient) userAuthServiceImpl := user.NewUserAuthServiceImpl(userAuthRepositoryImpl, sessionManager, loginService, sugaredLogger, userRepositoryImpl, roleGroupRepositoryImpl, userServiceImpl) environmentServiceImpl := cluster2.NewEnvironmentServiceImpl(environmentRepositoryImpl, clusterServiceImplExtended, sugaredLogger, k8sServiceImpl, k8sInformerFactoryImpl, userAuthServiceImpl, attributesRepositoryImpl) helmReleaseConfig, err := client3.GetHelmReleaseConfig() @@ -309,7 +314,7 @@ func InitializeApp() (*App, error) { return nil, err } versionServiceImpl := argocdServer.NewVersionServiceImpl(sugaredLogger, argoCDConnectionManagerImpl) - argoUserServiceImpl, err := argo.NewArgoUserServiceImpl(sugaredLogger, clusterServiceImplExtended, devtronSecretConfig, runtimeConfig, gitOpsConfigRepositoryImpl, argoCDConnectionManagerImpl, versionServiceImpl, k8sServiceImpl) + argoUserServiceImpl, err := argo.NewArgoUserServiceImpl(sugaredLogger, clusterServiceImplExtended, devtronSecretConfig, runtimeConfig, argoCDConnectionManagerImpl, versionServiceImpl, k8sServiceImpl, gitOpsConfigReadServiceImpl) if err != nil { return nil, err } @@ -328,10 +333,6 @@ func InitializeApp() (*App, error) { enforcerImpl := casbin.NewEnforcerImpl(syncedEnforcer, sessionManager, sugaredLogger) enforcerUtilImpl := rbac.NewEnforcerUtilImpl(sugaredLogger, teamRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, clusterRepositoryImpl, enforcerImpl) appStatusServiceImpl := appStatus2.NewAppStatusServiceImpl(appStatusRepositoryImpl, sugaredLogger, enforcerImpl, enforcerUtilImpl) - globalEnvVariables, err := util2.GetGlobalEnvVariables() - if err != nil { - return nil, err - } scopedVariableRepositoryImpl := repository6.NewScopedVariableRepository(db, sugaredLogger) qualifiersMappingRepositoryImpl, err := resourceQualifiers.NewQualifiersMappingRepositoryImpl(db, sugaredLogger) if err != nil { @@ -371,7 +372,6 @@ func InitializeApp() (*App, error) { Logger: sugaredLogger, } chartRefServiceImpl := chartRef.NewChartRefServiceImpl(sugaredLogger, chartRefRepositoryImpl, chartTemplateServiceImpl, utilMergeUtil) - gitOpsConfigReadServiceImpl := config.NewGitOpsConfigReadServiceImpl(sugaredLogger, gitOpsConfigRepositoryImpl, userServiceImpl, globalEnvVariables) gitCliUtil := git.NewGitCliUtil(sugaredLogger) gitFactory, err := git.NewGitFactory(sugaredLogger, gitOpsConfigRepositoryImpl, gitCliUtil) if err != nil { @@ -547,7 +547,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - appStoreDeploymentServiceImpl := service2.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, clusterInstalledAppsRepositoryImpl, appRepositoryImpl, appStoreDeploymentHelmServiceImpl, appStoreDeploymentArgoCdServiceImpl, environmentServiceImpl, clusterServiceImplExtended, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, installedAppVersionHistoryRepositoryImpl, gitOpsConfigRepositoryImpl, serviceDeploymentServiceTypeConfig, acdConfig, gitOpsConfigReadServiceImpl, gitOperationServiceImpl) + appStoreDeploymentServiceImpl := service2.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, clusterInstalledAppsRepositoryImpl, appRepositoryImpl, appStoreDeploymentHelmServiceImpl, appStoreDeploymentArgoCdServiceImpl, environmentServiceImpl, clusterServiceImplExtended, helmAppServiceImpl, appStoreDeploymentCommonServiceImpl, installedAppVersionHistoryRepositoryImpl, serviceDeploymentServiceTypeConfig, acdConfig, gitOpsConfigReadServiceImpl, gitOperationServiceImpl) k8sResourceHistoryRepositoryImpl := repository16.NewK8sResourceHistoryRepositoryImpl(db, sugaredLogger) k8sResourceHistoryServiceImpl := kubernetesResourceAuditLogs.Newk8sResourceHistoryServiceImpl(k8sResourceHistoryRepositoryImpl, sugaredLogger, appRepositoryImpl, environmentRepositoryImpl) ephemeralContainersRepositoryImpl := repository2.NewEphemeralContainersRepositoryImpl(db) @@ -557,7 +557,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - installedAppServiceImpl, err := service2.NewInstalledAppServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, teamRepositoryImpl, appRepositoryImpl, applicationServiceClientImpl, appStoreValuesServiceImpl, pubSubClientServiceImpl, chartGroupDeploymentRepositoryImpl, environmentServiceImpl, acdAuthConfig, gitOpsConfigRepositoryImpl, userServiceImpl, appStoreDeploymentFullModeServiceImpl, appStoreDeploymentServiceImpl, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, helmAppClientImpl, helmAppServiceImpl, appStatusServiceImpl, k8sServiceImpl, pipelineStatusTimelineServiceImpl, appStoreDeploymentCommonServiceImpl, k8sCommonServiceImpl, k8sApplicationServiceImpl, acdConfig, gitOpsConfigReadServiceImpl, gitOperationServiceImpl) + installedAppServiceImpl, err := service2.NewInstalledAppServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, environmentRepositoryImpl, teamRepositoryImpl, appRepositoryImpl, applicationServiceClientImpl, appStoreValuesServiceImpl, pubSubClientServiceImpl, chartGroupDeploymentRepositoryImpl, environmentServiceImpl, acdAuthConfig, userServiceImpl, appStoreDeploymentFullModeServiceImpl, appStoreDeploymentServiceImpl, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, helmAppClientImpl, helmAppServiceImpl, appStatusServiceImpl, k8sServiceImpl, pipelineStatusTimelineServiceImpl, appStoreDeploymentCommonServiceImpl, k8sCommonServiceImpl, k8sApplicationServiceImpl, acdConfig, gitOpsConfigReadServiceImpl, gitOperationServiceImpl) if err != nil { return nil, err } @@ -716,7 +716,7 @@ func InitializeApp() (*App, error) { return nil, err } providerIdentifierServiceImpl := providerIdentifier.NewProviderIdentifierServiceImpl(sugaredLogger) - telemetryEventClientImplExtended, err := telemetry.NewTelemetryEventClientImplExtended(sugaredLogger, httpClient, clusterServiceImplExtended, k8sServiceImpl, acdAuthConfig, environmentServiceImpl, userServiceImpl, appListingRepositoryImpl, posthogClient, ciPipelineRepositoryImpl, pipelineRepositoryImpl, gitOpsConfigRepositoryImpl, gitProviderRepositoryImpl, attributesRepositoryImpl, ssoLoginServiceImpl, appRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, materialRepositoryImpl, ciTemplateRepositoryImpl, chartRepositoryImpl, userAuditServiceImpl, ciBuildConfigServiceImpl, moduleRepositoryImpl, serverDataStoreServerDataStore, helmAppClientImpl, installedAppRepositoryImpl, userAttributesRepositoryImpl, providerIdentifierServiceImpl) + telemetryEventClientImplExtended, err := telemetry.NewTelemetryEventClientImplExtended(sugaredLogger, httpClient, clusterServiceImplExtended, k8sServiceImpl, acdAuthConfig, environmentServiceImpl, userServiceImpl, appListingRepositoryImpl, posthogClient, ciPipelineRepositoryImpl, pipelineRepositoryImpl, gitProviderRepositoryImpl, attributesRepositoryImpl, ssoLoginServiceImpl, appRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, materialRepositoryImpl, ciTemplateRepositoryImpl, chartRepositoryImpl, userAuditServiceImpl, ciBuildConfigServiceImpl, moduleRepositoryImpl, serverDataStoreServerDataStore, helmAppClientImpl, installedAppRepositoryImpl, userAttributesRepositoryImpl, providerIdentifierServiceImpl, gitOpsConfigReadServiceImpl) if err != nil { return nil, err } From 6e83a5c8f4779f239b65906dde8c58bddb11bd89 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 23 Jan 2024 14:29:03 +0530 Subject: [PATCH 36/64] refactored gitOpsRepository usages --- Wire.go | 2 - cmd/external-app/wire.go | 3 - pkg/deployment/gitOps/git/GitService.go | 37 +++++------ pkg/deployment/gitOps/git/GitServiceAzure.go | 45 +++---------- .../gitOps/git/GitServiceBitbucket.go | 63 +++---------------- pkg/deployment/gitOps/git/GitServiceGithub.go | 42 +++---------- pkg/deployment/gitOps/git/GitServiceGitlab.go | 23 ------- pkg/deployment/gitOps/wire_gitOps.go | 4 ++ pkg/gitops/GitOpsConfigService.go | 4 +- 9 files changed, 48 insertions(+), 175 deletions(-) diff --git a/Wire.go b/Wire.go index d3b14ac7cb..136cb0cf7e 100644 --- a/Wire.go +++ b/Wire.go @@ -637,8 +637,6 @@ func InitializeApp() (*App, error) { wire.Bind(new(restHandler.GitOpsConfigRestHandler), new(*restHandler.GitOpsConfigRestHandlerImpl)), gitops.NewGitOpsConfigServiceImpl, wire.Bind(new(gitops.GitOpsConfigService), new(*gitops.GitOpsConfigServiceImpl)), - repository.NewGitOpsConfigRepositoryImpl, - wire.Bind(new(repository.GitOpsConfigRepository), new(*repository.GitOpsConfigRepositoryImpl)), router.NewAttributesRouterImpl, wire.Bind(new(router.AttributesRouter), new(*router.AttributesRouterImpl)), diff --git a/cmd/external-app/wire.go b/cmd/external-app/wire.go index da0f16094b..fb504e2c7d 100644 --- a/cmd/external-app/wire.go +++ b/cmd/external-app/wire.go @@ -154,9 +154,6 @@ func InitializeApp() (*App, error) { wire.Bind(new(dashboardEvent.DashboardTelemetryRouter), new(*dashboardEvent.DashboardTelemetryRouterImpl)), - repository.NewGitOpsConfigRepositoryImpl, - wire.Bind(new(repository.GitOpsConfigRepository), new(*repository.GitOpsConfigRepositoryImpl)), - //binding argoUserService to helm via dummy implementation(HelmUserServiceImpl) argo.NewHelmUserServiceImpl, wire.Bind(new(argo.ArgoUserService), new(*argo.HelmUserServiceImpl)), diff --git a/pkg/deployment/gitOps/git/GitService.go b/pkg/deployment/gitOps/git/GitService.go index 6f0f3815d1..80d8b48cb3 100644 --- a/pkg/deployment/gitOps/git/GitService.go +++ b/pkg/deployment/gitOps/git/GitService.go @@ -56,15 +56,13 @@ type GitClient interface { GetRepoUrl(config *bean2.GitOpsConfigDto) (repoUrl string, err error) DeleteRepository(config *bean2.GitOpsConfigDto) error CreateReadme(config *bean2.GitOpsConfigDto) (string, error) - GetCommits(repoName, projectName string) ([]*GitCommitDto, error) } type GitFactory struct { - Client GitClient - GitService GitService - logger *zap.SugaredLogger - gitOpsRepository repository.GitOpsConfigRepository - gitCliUtil *GitCliUtil + Client GitClient + GitService GitService + logger *zap.SugaredLogger + gitCliUtil *GitCliUtil } type DetailedErrorGitOpsConfigActions struct { @@ -80,20 +78,20 @@ type GitCommitDto struct { CommitTime time.Time `json:"commitTime"` } -func (factory *GitFactory) Reload() error { +func (factory *GitFactory) Reload(gitOpsRepository repository.GitOpsConfigRepository) error { var err error start := time.Now() defer func() { util.TriggerGitOpsMetrics("Reload", "GitService", start, err) }() factory.logger.Infow("reloading gitops details") - cfg, err := GetGitConfig(factory.gitOpsRepository) + cfg, err := GetGitConfig(gitOpsRepository) if err != nil { return err } gitService := NewGitServiceImpl(cfg, factory.logger, factory.gitCliUtil) factory.GitService = gitService - client, err := NewGitOpsClient(cfg, factory.logger, gitService, factory.gitOpsRepository) + client, err := NewGitOpsClient(cfg, factory.logger, gitService) if err != nil { return err } @@ -158,7 +156,7 @@ func (factory *GitFactory) NewClientForValidation(gitOpsConfig *bean2.GitOpsConf } gitService := NewGitServiceImpl(cfg, factory.logger, factory.gitCliUtil) //factory.GitService = GitService - client, err := NewGitOpsClient(cfg, factory.logger, gitService, factory.gitOpsRepository) + client, err := NewGitOpsClient(cfg, factory.logger, gitService) if err != nil { return client, gitService, err } @@ -174,16 +172,15 @@ func NewGitFactory(logger *zap.SugaredLogger, gitOpsRepository repository.GitOps return nil, err } gitService := NewGitServiceImpl(cfg, logger, gitCliUtil) - client, err := NewGitOpsClient(cfg, logger, gitService, gitOpsRepository) + client, err := NewGitOpsClient(cfg, logger, gitService) if err != nil { logger.Errorw("error in creating gitOps client", "err", err, "gitProvider", cfg.GitProvider) } return &GitFactory{ - Client: client, - logger: logger, - GitService: gitService, - gitOpsRepository: gitOpsRepository, - gitCliUtil: gitCliUtil, + Client: client, + logger: logger, + GitService: gitService, + gitCliUtil: gitCliUtil, }, nil } @@ -231,18 +228,18 @@ func GetGitConfig(gitOpsRepository repository.GitOpsConfigRepository) (*GitConfi return cfg, err } -func NewGitOpsClient(config *GitConfig, logger *zap.SugaredLogger, gitService GitService, gitOpsConfigRepository repository.GitOpsConfigRepository) (GitClient, error) { +func NewGitOpsClient(config *GitConfig, logger *zap.SugaredLogger, gitService GitService) (GitClient, error) { if config.GitProvider == GITLAB_PROVIDER { gitLabClient, err := NewGitLabClient(config, logger, gitService) return gitLabClient, err } else if config.GitProvider == GITHUB_PROVIDER { - gitHubClient, err := NewGithubClient(config.GitHost, config.GitToken, config.GithubOrganization, logger, gitService, gitOpsConfigRepository) + gitHubClient, err := NewGithubClient(config.GitHost, config.GitToken, config.GithubOrganization, logger, gitService) return gitHubClient, err } else if config.GitProvider == AZURE_DEVOPS_PROVIDER { - gitAzureClient, err := NewGitAzureClient(config.AzureToken, config.GitHost, config.AzureProject, logger, gitService, gitOpsConfigRepository) + gitAzureClient, err := NewGitAzureClient(config.AzureToken, config.GitHost, config.AzureProject, logger, gitService) return gitAzureClient, err } else if config.GitProvider == BITBUCKET_PROVIDER { - gitBitbucketClient := NewGitBitbucketClient(config.GitUserName, config.GitToken, config.GitHost, logger, gitService, gitOpsConfigRepository) + gitBitbucketClient := NewGitBitbucketClient(config.GitUserName, config.GitToken, config.GitHost, logger, gitService) return gitBitbucketClient, nil } else { logger.Errorw("no gitops config provided, gitops will not work ") diff --git a/pkg/deployment/gitOps/git/GitServiceAzure.go b/pkg/deployment/gitOps/git/GitServiceAzure.go index 1bb7dfbbc7..b1fd155d1e 100644 --- a/pkg/deployment/gitOps/git/GitServiceAzure.go +++ b/pkg/deployment/gitOps/git/GitServiceAzure.go @@ -4,7 +4,6 @@ import ( "context" "fmt" bean2 "github.com/devtron-labs/devtron/api/bean" - "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/microsoft/azure-devops-go-api/azuredevops" "github.com/microsoft/azure-devops-go-api/azuredevops/git" "go.uber.org/zap" @@ -13,11 +12,10 @@ import ( ) type GitAzureClient struct { - client *git.Client - logger *zap.SugaredLogger - project string - gitService GitService - gitOpsConfigRepository repository.GitOpsConfigRepository + client *git.Client + logger *zap.SugaredLogger + project string + gitService GitService } func (impl GitAzureClient) GetRepoUrl(config *bean2.GitOpsConfigDto) (repoUrl string, err error) { @@ -31,8 +29,7 @@ func (impl GitAzureClient) GetRepoUrl(config *bean2.GitOpsConfigDto) (repoUrl st } } -func NewGitAzureClient(token string, host string, project string, logger *zap.SugaredLogger, gitService GitService, - gitOpsConfigRepository repository.GitOpsConfigRepository) (GitAzureClient, error) { +func NewGitAzureClient(token string, host string, project string, logger *zap.SugaredLogger, gitService GitService) (GitAzureClient, error) { ctx := context.Background() // Create a connection to your organization connection := azuredevops.NewPatConnection(host, token) @@ -42,11 +39,10 @@ func NewGitAzureClient(token string, host string, project string, logger *zap.Su logger.Errorw("error in creating azure gitops client, gitops related operation might fail", "err", err) } return GitAzureClient{ - client: &coreClient, - project: project, - logger: logger, - gitService: gitService, - gitOpsConfigRepository: gitOpsConfigRepository, + client: &coreClient, + project: project, + logger: logger, + gitService: gitService, }, err } func (impl GitAzureClient) DeleteRepository(config *bean2.GitOpsConfigDto) error { @@ -298,26 +294,3 @@ func (impl GitAzureClient) ensureProjectAvailabilityOnSsh(projectName string, re } return false, nil } - -func (impl GitAzureClient) GetCommits(repoName, projectName string) ([]*GitCommitDto, error) { - azureClient := *impl.client - getCommitsArgs := git.GetCommitsArgs{ - RepositoryId: &repoName, - Project: &projectName, - } - gitCommits, err := azureClient.GetCommits(context.Background(), getCommitsArgs) - if err != nil { - impl.logger.Errorw("error in getting commits", "err", err, "repoName", repoName, "projectName", projectName) - return nil, err - } - var gitCommitsDto []*GitCommitDto - for _, gitCommit := range *gitCommits { - gitCommitDto := &GitCommitDto{ - CommitHash: *gitCommit.CommitId, - AuthorName: *gitCommit.Author.Name, - CommitTime: gitCommit.Author.Date.Time, - } - gitCommitsDto = append(gitCommitsDto, gitCommitDto) - } - return gitCommitsDto, nil -} diff --git a/pkg/deployment/gitOps/git/GitServiceBitbucket.go b/pkg/deployment/gitOps/git/GitServiceBitbucket.go index ec9296f84f..a7f6cd1693 100644 --- a/pkg/deployment/gitOps/git/GitServiceBitbucket.go +++ b/pkg/deployment/gitOps/git/GitServiceBitbucket.go @@ -3,8 +3,6 @@ package git import ( "fmt" bean2 "github.com/devtron-labs/devtron/api/bean" - "github.com/devtron-labs/devtron/internal/sql/repository" - "github.com/go-pg/pg" "github.com/ktrysmt/go-bitbucket" "go.uber.org/zap" "io/ioutil" @@ -22,21 +20,18 @@ const ( ) type GitBitbucketClient struct { - client *bitbucket.Client - logger *zap.SugaredLogger - gitService GitService - gitOpsConfigRepository repository.GitOpsConfigRepository + client *bitbucket.Client + logger *zap.SugaredLogger + gitService GitService } -func NewGitBitbucketClient(username, token, host string, logger *zap.SugaredLogger, gitService GitService, - gitOpsConfigRepository repository.GitOpsConfigRepository) GitBitbucketClient { +func NewGitBitbucketClient(username, token, host string, logger *zap.SugaredLogger, gitService GitService) GitBitbucketClient { coreClient := bitbucket.NewBasicAuth(username, token) logger.Infow("bitbucket client created", "clientDetails", coreClient) return GitBitbucketClient{ - client: coreClient, - logger: logger, - gitService: gitService, - gitOpsConfigRepository: gitOpsConfigRepository, + client: coreClient, + logger: logger, + gitService: gitService, } } @@ -256,47 +251,3 @@ func (impl GitBitbucketClient) CommitValues(config *ChartConfig, gitOpsConfig *b } return commitHash, commitTime, nil } - -func (impl GitBitbucketClient) GetCommits(repoName, projectName string) ([]*GitCommitDto, error) { - gitOpsConfigBitbucket, err := impl.gitOpsConfigRepository.GetGitOpsConfigByProvider(BITBUCKET_PROVIDER) - if err != nil { - if err == pg.ErrNoRows { - gitOpsConfigBitbucket = &repository.GitOpsConfig{} - gitOpsConfigBitbucket.BitBucketWorkspaceId = "" - gitOpsConfigBitbucket.BitBucketProjectKey = "" - } else { - impl.logger.Errorw("error in fetching gitOps bitbucket config", "err", err) - return nil, err - } - } - bitbucketWorkspaceId := gitOpsConfigBitbucket.BitBucketWorkspaceId - bitbucketClient := impl.client - getCommitsOptions := &bitbucket.CommitsOptions{ - RepoSlug: repoName, - Owner: bitbucketWorkspaceId, - Branchortag: "master", - } - gitCommitsIf, err := bitbucketClient.Repositories.Commits.GetCommits(getCommitsOptions) - if err != nil { - impl.logger.Errorw("error in getting commits", "err", err, "repoName", repoName) - return nil, err - } - - gitCommits := gitCommitsIf.(map[string]interface{})["values"].([]interface{}) - var gitCommitsDto []*GitCommitDto - for _, gitCommit := range gitCommits { - - commitHash := gitCommit.(map[string]string)["hash"] - commitTime, err := time.Parse(time.RFC3339, gitCommit.(map[string]interface{})["date"].(string)) - if err != nil { - impl.logger.Errorw("error in getting commitTime", "err", err, "gitCommit", gitCommit) - return nil, err - } - gitCommitDto := &GitCommitDto{ - CommitHash: commitHash, - CommitTime: commitTime, - } - gitCommitsDto = append(gitCommitsDto, gitCommitDto) - } - return gitCommitsDto, nil -} diff --git a/pkg/deployment/gitOps/git/GitServiceGithub.go b/pkg/deployment/gitOps/git/GitServiceGithub.go index b7cb50c927..0969370cd4 100644 --- a/pkg/deployment/gitOps/git/GitServiceGithub.go +++ b/pkg/deployment/gitOps/git/GitServiceGithub.go @@ -4,7 +4,6 @@ import ( "context" "fmt" bean2 "github.com/devtron-labs/devtron/api/bean" - "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/google/go-github/github" "go.uber.org/zap" "golang.org/x/oauth2" @@ -16,17 +15,14 @@ import ( ) type GitHubClient struct { - client *github.Client - - //config *GitConfig - logger *zap.SugaredLogger - org string - gitService GitService - gitOpsConfigRepository repository.GitOpsConfigRepository + client *github.Client + logger *zap.SugaredLogger + org string + gitService GitService } func NewGithubClient(host string, token string, org string, logger *zap.SugaredLogger, - gitService GitService, gitOpsConfigRepository repository.GitOpsConfigRepository) (GitHubClient, error) { + gitService GitService) (GitHubClient, error) { ctx := context.Background() httpTransport := &http2.Transport{} httpClient := &http2.Client{Transport: httpTransport} @@ -52,11 +48,10 @@ func NewGithubClient(host string, token string, org string, logger *zap.SugaredL } return GitHubClient{ - client: client, - org: org, - logger: logger, - gitService: gitService, - gitOpsConfigRepository: gitOpsConfigRepository, + client: client, + org: org, + logger: logger, + gitService: gitService, }, err } func (impl GitHubClient) DeleteRepository(config *bean2.GitOpsConfigDto) error { @@ -244,22 +239,3 @@ func (impl GitHubClient) ensureProjectAvailabilityOnSsh(projectName string, repo } return false, nil } - -func (impl GitHubClient) GetCommits(repoName, projectName string) ([]*GitCommitDto, error) { - githubClient := impl.client - gitCommits, _, err := githubClient.Repositories.ListCommits(context.Background(), impl.org, repoName, &github.CommitsListOptions{}) - if err != nil { - impl.logger.Errorw("error in getting commits", "err", err, "repoName", repoName) - return nil, err - } - var gitCommitsDto []*GitCommitDto - for _, gitCommit := range gitCommits { - gitCommitDto := &GitCommitDto{ - CommitHash: gitCommit.GetSHA(), - AuthorName: *gitCommit.Author.Name, - CommitTime: *gitCommit.Commit.Author.Date, - } - gitCommitsDto = append(gitCommitsDto, gitCommitDto) - } - return gitCommitsDto, nil -} diff --git a/pkg/deployment/gitOps/git/GitServiceGitlab.go b/pkg/deployment/gitOps/git/GitServiceGitlab.go index 4f8e210698..fcdc4f8567 100644 --- a/pkg/deployment/gitOps/git/GitServiceGitlab.go +++ b/pkg/deployment/gitOps/git/GitServiceGitlab.go @@ -269,26 +269,3 @@ func (impl GitLabClient) CommitValues(config *ChartConfig, gitOpsConfig *bean2.G } return c.ID, *c.AuthoredDate, err } - -func (impl GitLabClient) GetCommits(repoName, projectName string) ([]*GitCommitDto, error) { - gitlabClient := impl.client - branch := "master" - listCommitOptions := &gitlab.ListCommitsOptions{ - RefName: &branch, - } - gitCommits, _, err := gitlabClient.Commits.ListCommits(fmt.Sprintf("%s/%s", impl.config.GitlabGroupPath, repoName), listCommitOptions) - if err != nil { - impl.logger.Errorw("error in getting commits", "err", err, "repoName", repoName) - return nil, err - } - var gitCommitsDto []*GitCommitDto - for _, gitCommit := range gitCommits { - gitCommitDto := &GitCommitDto{ - CommitHash: gitCommit.String(), - AuthorName: gitCommit.AuthorName, - CommitTime: *gitCommit.AuthoredDate, - } - gitCommitsDto = append(gitCommitsDto, gitCommitDto) - } - return gitCommitsDto, nil -} diff --git a/pkg/deployment/gitOps/wire_gitOps.go b/pkg/deployment/gitOps/wire_gitOps.go index a0b7daa517..7c2df27271 100644 --- a/pkg/deployment/gitOps/wire_gitOps.go +++ b/pkg/deployment/gitOps/wire_gitOps.go @@ -1,12 +1,16 @@ package gitOps import ( + "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" "github.com/google/wire" ) var GitOpsWireSet = wire.NewSet( + repository.NewGitOpsConfigRepositoryImpl, + wire.Bind(new(repository.GitOpsConfigRepository), new(*repository.GitOpsConfigRepositoryImpl)), + config.NewGitOpsConfigReadServiceImpl, wire.Bind(new(config.GitOpsConfigReadService), new(*config.GitOpsConfigReadServiceImpl)), diff --git a/pkg/gitops/GitOpsConfigService.go b/pkg/gitops/GitOpsConfigService.go index 06e8f7052e..5925bf06de 100644 --- a/pkg/gitops/GitOpsConfigService.go +++ b/pkg/gitops/GitOpsConfigService.go @@ -348,7 +348,7 @@ func (impl *GitOpsConfigServiceImpl) CreateGitOpsConfig(ctx context.Context, req return nil, err } - err = impl.gitFactory.Reload() + err = impl.gitFactory.Reload(impl.gitOpsRepository) if err != nil { return nil, err } @@ -523,7 +523,7 @@ func (impl *GitOpsConfigServiceImpl) UpdateGitOpsConfig(request *bean2.GitOpsCon if err != nil { return err } - err = impl.gitFactory.Reload() + err = impl.gitFactory.Reload(impl.gitOpsRepository) if err != nil { return err } From d6165740c0d1e374c06e88f0f907ae3d9ab7e607 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 23 Jan 2024 15:59:45 +0530 Subject: [PATCH 37/64] gitlab client creation refactoring --- pkg/deployment/gitOps/git/GitService.go | 28 ++----------- pkg/deployment/gitOps/git/GitServiceGitlab.go | 42 +++++++++++-------- .../deployment/gitOps/git}/GitService_test.go | 14 +++---- 3 files changed, 36 insertions(+), 48 deletions(-) rename {internal/util => pkg/deployment/gitOps/git}/GitService_test.go (74%) diff --git a/pkg/deployment/gitOps/git/GitService.go b/pkg/deployment/gitOps/git/GitService.go index 80d8b48cb3..20105abe01 100644 --- a/pkg/deployment/gitOps/git/GitService.go +++ b/pkg/deployment/gitOps/git/GitService.go @@ -21,7 +21,6 @@ import ( "context" "fmt" "github.com/devtron-labs/devtron/util" - "net/url" "path/filepath" "time" @@ -72,12 +71,6 @@ type DetailedErrorGitOpsConfigActions struct { DeleteRepoFailed bool `json:"deleteRepoFailed"` } -type GitCommitDto struct { - CommitHash string `json:"commitHash"` - AuthorName string `json:"authorName"` - CommitTime time.Time `json:"commitTime"` -} - func (factory *GitFactory) Reload(gitOpsRepository repository.GitOpsConfigRepository) error { var err error start := time.Now() @@ -102,27 +95,14 @@ func (factory *GitFactory) Reload(gitOpsRepository repository.GitOpsConfigReposi func (factory *GitFactory) GetGitLabGroupPath(gitOpsConfig *bean2.GitOpsConfigDto) (string, error) { start := time.Now() - var gitLabClient *gitlab.Client var err error defer func() { util.TriggerGitOpsMetrics("GetGitLabGroupPath", "GitService", start, err) }() - if len(gitOpsConfig.Host) > 0 { - _, err = url.ParseRequestURI(gitOpsConfig.Host) - if err != nil { - return "", err - } - gitLabClient, err = gitlab.NewClient(gitOpsConfig.Token, gitlab.WithBaseURL(gitOpsConfig.Host)) - if err != nil { - factory.logger.Errorw("error in getting new gitlab client", "err", err) - return "", err - } - } else { - gitLabClient, err = gitlab.NewClient(gitOpsConfig.Token) - if err != nil { - factory.logger.Errorw("error in getting new gitlab client", "err", err) - return "", err - } + gitLabClient, err := CreateGitlabClient(gitOpsConfig.Host, gitOpsConfig.Token) + if err != nil { + factory.logger.Errorw("error in creating gitlab client", "err", err) + return "", err } group, _, err := gitLabClient.Groups.GetGroup(gitOpsConfig.GitLabGroupId, &gitlab.GetGroupOptions{}) if err != nil { diff --git a/pkg/deployment/gitOps/git/GitServiceGitlab.go b/pkg/deployment/gitOps/git/GitServiceGitlab.go index fcdc4f8567..bf4a69cc63 100644 --- a/pkg/deployment/gitOps/git/GitServiceGitlab.go +++ b/pkg/deployment/gitOps/git/GitServiceGitlab.go @@ -19,24 +19,11 @@ type GitLabClient struct { } func NewGitLabClient(config *GitConfig, logger *zap.SugaredLogger, gitService GitService) (GitClient, error) { - var gitLabClient *gitlab.Client - var err error - if len(config.GitHost) > 0 { - _, err = url.ParseRequestURI(config.GitHost) - if err != nil { - return nil, err - } - gitLabClient, err = gitlab.NewClient(config.GitToken, gitlab.WithBaseURL(config.GitHost)) - if err != nil { - return nil, err - } - } else { - gitLabClient, err = gitlab.NewClient(config.GitToken) - if err != nil { - return nil, err - } + gitLabClient, err := CreateGitlabClient(config.GitHost, config.GitToken) + if err != nil { + logger.Errorw("error in creating gitlab client", "err", err) + return nil, err } - gitlabGroupId := "" if len(config.GitlabGroupId) > 0 { if _, err := strconv.Atoi(config.GitlabGroupId); err == nil { @@ -82,6 +69,27 @@ func NewGitLabClient(config *GitConfig, logger *zap.SugaredLogger, gitService Gi }, nil } +func CreateGitlabClient(host, token string) (*gitlab.Client, error) { + var gitLabClient *gitlab.Client + var err error + if len(host) > 0 { + _, err = url.ParseRequestURI(host) + if err != nil { + return nil, err + } + gitLabClient, err = gitlab.NewClient(token, gitlab.WithBaseURL(host)) + if err != nil { + return nil, err + } + } else { + gitLabClient, err = gitlab.NewClient(token) + if err != nil { + return nil, err + } + } + return gitLabClient, err +} + func (impl GitLabClient) DeleteRepository(config *bean2.GitOpsConfigDto) error { err := impl.DeleteProject(config.GitRepoName) if err != nil { diff --git a/internal/util/GitService_test.go b/pkg/deployment/gitOps/git/GitService_test.go similarity index 74% rename from internal/util/GitService_test.go rename to pkg/deployment/gitOps/git/GitService_test.go index 10eee631a3..f23da2212b 100644 --- a/internal/util/GitService_test.go +++ b/pkg/deployment/gitOps/git/GitService_test.go @@ -1,17 +1,17 @@ -package util +package git import ( "github.com/devtron-labs/devtron/api/bean" - "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" + "github.com/devtron-labs/devtron/internal/util" "testing" ) -func getTestGithubClient() git.GitHubClient { - logger, err := NewSugardLogger() - gitCliUtl := git.NewGitCliUtil(logger) - gitService := git.NewGitServiceImpl(&git.GitConfig{GitToken: "", GitUserName: "nishant"}, logger, gitCliUtl) +func getTestGithubClient() GitHubClient { + logger, err := util.NewSugardLogger() + gitCliUtl := NewGitCliUtil(logger) + gitService := NewGitServiceImpl(&GitConfig{GitToken: "", GitUserName: "nishant"}, logger, gitCliUtl) - githubClient, err := git.NewGithubClient("", "", "test-org", logger, gitService, nil) + githubClient, err := NewGithubClient("", "", "test-org", logger, gitService) if err != nil { panic(err) } From bde77ef545806258a79ab6b35b0c65ef4739bca0 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 23 Jan 2024 16:11:12 +0530 Subject: [PATCH 38/64] renamed util/ChartService --- internal/util/{ChartService.go => ChartTemplateService.go} | 0 .../util/{ChartService_test.go => ChartTemplateService_test.go} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename internal/util/{ChartService.go => ChartTemplateService.go} (100%) rename internal/util/{ChartService_test.go => ChartTemplateService_test.go} (100%) diff --git a/internal/util/ChartService.go b/internal/util/ChartTemplateService.go similarity index 100% rename from internal/util/ChartService.go rename to internal/util/ChartTemplateService.go diff --git a/internal/util/ChartService_test.go b/internal/util/ChartTemplateService_test.go similarity index 100% rename from internal/util/ChartService_test.go rename to internal/util/ChartTemplateService_test.go From 9aa630b6c0aa890d88510964adc0769fa735657c Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Wed, 24 Jan 2024 17:19:33 +0530 Subject: [PATCH 39/64] reverted renaming changes --- .../deployment-chart_1-0-0/app-values.yaml | 4 ++-- .../deployment-chart_1-1-0/app-values.yaml | 4 ++-- .../deployment-chart_4-18-0/app-values.yaml | 4 ++-- .../deployment-chart_4-19-0/app-values.yaml | 4 ++-- .../reference-chart_3-12-0/app-values.yaml | 4 ++-- .../reference-chart_3-13-0/app-values.yaml | 4 ++-- .../reference-chart_4-11-0/app-values.yaml | 4 ++-- .../reference-chart_4-12-0/app-values.yaml | 4 ++-- .../reference-chart_4-13-0/app-values.yaml | 4 ++-- .../reference-chart_4-14-0/app-values.yaml | 4 ++-- .../reference-chart_4-15-0/app-values.yaml | 4 ++-- .../reference-chart_4-16-0/app-values.yaml | 4 ++-- .../reference-chart_4-17-0/app-values.yaml | 4 ++-- .../reference-chart_4-18-0/app-values.yaml | 4 ++-- .../reference-chart_5-0-0/app-values.yaml | 4 ++-- .../statefulset-chart_4-18-0/app-values.yaml | 4 ++-- .../statefulset-chart_4-19-0/app-values.yaml | 4 ++-- .../statefulset-chart_5-0-0/app-values.yaml | 4 ++-- 18 files changed, 36 insertions(+), 36 deletions(-) diff --git a/scripts/devtron-reference-helm-charts/deployment-chart_1-0-0/app-values.yaml b/scripts/devtron-reference-helm-charts/deployment-chart_1-0-0/app-values.yaml index a5bebb5c56..0dce06c558 100644 --- a/scripts/devtron-reference-helm-charts/deployment-chart_1-0-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/deployment-chart_1-0-0/app-values.yaml @@ -308,5 +308,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.git" -# - "bar.git" +# - "foo.remote" +# - "bar.remote" diff --git a/scripts/devtron-reference-helm-charts/deployment-chart_1-1-0/app-values.yaml b/scripts/devtron-reference-helm-charts/deployment-chart_1-1-0/app-values.yaml index 664b444031..bbebc7acab 100644 --- a/scripts/devtron-reference-helm-charts/deployment-chart_1-1-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/deployment-chart_1-1-0/app-values.yaml @@ -449,5 +449,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.git" -# - "bar.git" +# - "foo.remote" +# - "bar.remote" diff --git a/scripts/devtron-reference-helm-charts/deployment-chart_4-18-0/app-values.yaml b/scripts/devtron-reference-helm-charts/deployment-chart_4-18-0/app-values.yaml index 96bfcafec7..ff562cb579 100644 --- a/scripts/devtron-reference-helm-charts/deployment-chart_4-18-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/deployment-chart_4-18-0/app-values.yaml @@ -508,5 +508,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.git" -# - "bar.git" +# - "foo.remote" +# - "bar.remote" diff --git a/scripts/devtron-reference-helm-charts/deployment-chart_4-19-0/app-values.yaml b/scripts/devtron-reference-helm-charts/deployment-chart_4-19-0/app-values.yaml index 88daa46112..ce73c2af57 100644 --- a/scripts/devtron-reference-helm-charts/deployment-chart_4-19-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/deployment-chart_4-19-0/app-values.yaml @@ -514,5 +514,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.git" -# - "bar.git" +# - "foo.remote" +# - "bar.remote" diff --git a/scripts/devtron-reference-helm-charts/reference-chart_3-12-0/app-values.yaml b/scripts/devtron-reference-helm-charts/reference-chart_3-12-0/app-values.yaml index f524a92391..85f565f40a 100644 --- a/scripts/devtron-reference-helm-charts/reference-chart_3-12-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/reference-chart_3-12-0/app-values.yaml @@ -233,5 +233,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.git" -# - "bar.git" +# - "foo.remote" +# - "bar.remote" diff --git a/scripts/devtron-reference-helm-charts/reference-chart_3-13-0/app-values.yaml b/scripts/devtron-reference-helm-charts/reference-chart_3-13-0/app-values.yaml index f524a92391..85f565f40a 100644 --- a/scripts/devtron-reference-helm-charts/reference-chart_3-13-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/reference-chart_3-13-0/app-values.yaml @@ -233,5 +233,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.git" -# - "bar.git" +# - "foo.remote" +# - "bar.remote" diff --git a/scripts/devtron-reference-helm-charts/reference-chart_4-11-0/app-values.yaml b/scripts/devtron-reference-helm-charts/reference-chart_4-11-0/app-values.yaml index 2771ba40ed..a9eca0e03a 100644 --- a/scripts/devtron-reference-helm-charts/reference-chart_4-11-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/reference-chart_4-11-0/app-values.yaml @@ -276,5 +276,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.git" -# - "bar.git" +# - "foo.remote" +# - "bar.remote" diff --git a/scripts/devtron-reference-helm-charts/reference-chart_4-12-0/app-values.yaml b/scripts/devtron-reference-helm-charts/reference-chart_4-12-0/app-values.yaml index e9fdbd4d14..a3022d7bb8 100644 --- a/scripts/devtron-reference-helm-charts/reference-chart_4-12-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/reference-chart_4-12-0/app-values.yaml @@ -289,5 +289,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.git" -# - "bar.git" +# - "foo.remote" +# - "bar.remote" diff --git a/scripts/devtron-reference-helm-charts/reference-chart_4-13-0/app-values.yaml b/scripts/devtron-reference-helm-charts/reference-chart_4-13-0/app-values.yaml index e9fdbd4d14..a3022d7bb8 100644 --- a/scripts/devtron-reference-helm-charts/reference-chart_4-13-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/reference-chart_4-13-0/app-values.yaml @@ -289,5 +289,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.git" -# - "bar.git" +# - "foo.remote" +# - "bar.remote" diff --git a/scripts/devtron-reference-helm-charts/reference-chart_4-14-0/app-values.yaml b/scripts/devtron-reference-helm-charts/reference-chart_4-14-0/app-values.yaml index 87327b1d51..d73eee8c80 100644 --- a/scripts/devtron-reference-helm-charts/reference-chart_4-14-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/reference-chart_4-14-0/app-values.yaml @@ -293,5 +293,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.git" -# - "bar.git" +# - "foo.remote" +# - "bar.remote" diff --git a/scripts/devtron-reference-helm-charts/reference-chart_4-15-0/app-values.yaml b/scripts/devtron-reference-helm-charts/reference-chart_4-15-0/app-values.yaml index 92eb571112..ff44a3679a 100644 --- a/scripts/devtron-reference-helm-charts/reference-chart_4-15-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/reference-chart_4-15-0/app-values.yaml @@ -306,5 +306,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.git" -# - "bar.git" +# - "foo.remote" +# - "bar.remote" diff --git a/scripts/devtron-reference-helm-charts/reference-chart_4-16-0/app-values.yaml b/scripts/devtron-reference-helm-charts/reference-chart_4-16-0/app-values.yaml index a93d0fb412..fe18509efd 100644 --- a/scripts/devtron-reference-helm-charts/reference-chart_4-16-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/reference-chart_4-16-0/app-values.yaml @@ -308,5 +308,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.git" -# - "bar.git" +# - "foo.remote" +# - "bar.remote" diff --git a/scripts/devtron-reference-helm-charts/reference-chart_4-17-0/app-values.yaml b/scripts/devtron-reference-helm-charts/reference-chart_4-17-0/app-values.yaml index 80ccc7cb32..a621f49a79 100644 --- a/scripts/devtron-reference-helm-charts/reference-chart_4-17-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/reference-chart_4-17-0/app-values.yaml @@ -360,5 +360,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.git" -# - "bar.git" +# - "foo.remote" +# - "bar.remote" diff --git a/scripts/devtron-reference-helm-charts/reference-chart_4-18-0/app-values.yaml b/scripts/devtron-reference-helm-charts/reference-chart_4-18-0/app-values.yaml index 8079c1e887..f4c8cef663 100644 --- a/scripts/devtron-reference-helm-charts/reference-chart_4-18-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/reference-chart_4-18-0/app-values.yaml @@ -424,5 +424,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.git" -# - "bar.git" +# - "foo.remote" +# - "bar.remote" diff --git a/scripts/devtron-reference-helm-charts/reference-chart_5-0-0/app-values.yaml b/scripts/devtron-reference-helm-charts/reference-chart_5-0-0/app-values.yaml index fd93c244d8..0555153aee 100644 --- a/scripts/devtron-reference-helm-charts/reference-chart_5-0-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/reference-chart_5-0-0/app-values.yaml @@ -423,5 +423,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.git" -# - "bar.git" +# - "foo.remote" +# - "bar.remote" diff --git a/scripts/devtron-reference-helm-charts/statefulset-chart_4-18-0/app-values.yaml b/scripts/devtron-reference-helm-charts/statefulset-chart_4-18-0/app-values.yaml index 1a8063c237..f1d80a0ea3 100644 --- a/scripts/devtron-reference-helm-charts/statefulset-chart_4-18-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/statefulset-chart_4-18-0/app-values.yaml @@ -393,5 +393,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.git" -# - "bar.git" +# - "foo.remote" +# - "bar.remote" diff --git a/scripts/devtron-reference-helm-charts/statefulset-chart_4-19-0/app-values.yaml b/scripts/devtron-reference-helm-charts/statefulset-chart_4-19-0/app-values.yaml index f7edfb417e..ad27515a44 100644 --- a/scripts/devtron-reference-helm-charts/statefulset-chart_4-19-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/statefulset-chart_4-19-0/app-values.yaml @@ -397,5 +397,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.git" -# - "bar.git" +# - "foo.remote" +# - "bar.remote" diff --git a/scripts/devtron-reference-helm-charts/statefulset-chart_5-0-0/app-values.yaml b/scripts/devtron-reference-helm-charts/statefulset-chart_5-0-0/app-values.yaml index 4a057c104f..1b1912aa68 100644 --- a/scripts/devtron-reference-helm-charts/statefulset-chart_5-0-0/app-values.yaml +++ b/scripts/devtron-reference-helm-charts/statefulset-chart_5-0-0/app-values.yaml @@ -384,5 +384,5 @@ hostAliases: [] # - "bar.local" # - ip: "10.1.2.3" # hostnames: -# - "foo.git" -# - "bar.git" +# - "foo.remote" +# - "bar.remote" From 9961fd0cf9f3533e77eea5ccc421c8f70a8636dd Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Wed, 24 Jan 2024 17:22:22 +0530 Subject: [PATCH 40/64] reverted renaming changes --- scripts/sql/49_plugin_integration.up.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/sql/49_plugin_integration.up.sql b/scripts/sql/49_plugin_integration.up.sql index 2fd34bc5b9..e52a73ccbe 100644 --- a/scripts/sql/49_plugin_integration.up.sql +++ b/scripts/sql/49_plugin_integration.up.sql @@ -307,11 +307,11 @@ then export GOCACHE=/usr/local/go/cache export PATH=$PATH:/usr/local/go/bin go install go.k6.io/xk6/cmd/xk6@latest - xk6 build --with github.com/grafana/xk6-output-prometheus-git + xk6 build --with github.com/grafana/xk6-output-prometheus-remote K6_PROMETHEUS_USER=$PrometheusUsername \ K6_PROMETHEUS_PASSWORD=$PrometheusApiKey \ K6_PROMETHEUS_REMOTE_URL=$PrometheusRemoteWriteEndpoint \ - ./k6 run $PathToScript -o output-prometheus-git + ./k6 run $PathToScript -o output-prometheus-remote elif [ $OutputType == "LOG" ] then docker pull grafana/k6 @@ -343,7 +343,7 @@ INSERT INTO "public"."plugin_step_variable" ("id", "plugin_step_id", "name", "fo ('1', '1','RelativePathToScript','STRING','checkout path + script path along with script name','t','f','INPUT','NEW','/./script.js','1','f','now()', '1', 'now()', '1'), ('2', '1','PrometheusUsername','STRING','username of prometheus account','t','t','INPUT','NEW',null, '1' ,'f','now()', '1', 'now()', '1'), ('3', '1','PrometheusApiKey','STRING','api key of prometheus account','t','t','INPUT','NEW',null, '1','f','now()', '1', 'now()', '1'), -('4', '1','PrometheusRemoteWriteEndpoint','STRING','git write endpoint of prometheus account','t','t','INPUT','NEW',null, '1','f','now()', '1', 'now()', '1'), +('4', '1','PrometheusRemoteWriteEndpoint','STRING','remote write endpoint of prometheus account','t','t','INPUT','NEW',null, '1','f','now()', '1', 'now()', '1'), ('5', '1','OutputType','STRING','output type - LOG or PROMETHEUS','t','f','INPUT','NEW','LOG', '1','f','now()', '1', 'now()', '1'), ('6', '2','SonarqubeProjectKey','STRING','project key of grafana sonarqube account','t','t','INPUT','NEW',null, '1', 'f','now()', '1', 'now()', '1'), ('7', '2','SonarqubeApiKey','STRING','api key of sonarqube account','t','t','INPUT','NEW',null, '1', 'f','now()', '1', 'now()', '1'), From 5e128a06c434a9066a3a2662d783296dee919331 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Wed, 24 Jan 2024 17:41:23 +0530 Subject: [PATCH 41/64] reverted renaming changes --- .../service/AppStoreDeploymentService_test.go | 4 ++-- pkg/deployment/gitOps/git/GitCliUtil.go | 2 +- pkg/pipeline/DeploymentPipelineConfigService.go | 2 +- pkg/pipeline/PipelineStageServiceIT_test.go | 12 ++++++------ util/RequestUtil.go | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/appStore/deployment/service/AppStoreDeploymentService_test.go b/pkg/appStore/deployment/service/AppStoreDeploymentService_test.go index 01aabf55f1..79a6fd1f9f 100644 --- a/pkg/appStore/deployment/service/AppStoreDeploymentService_test.go +++ b/pkg/appStore/deployment/service/AppStoreDeploymentService_test.go @@ -43,7 +43,7 @@ func TestAppStoreDeploymentService(t *testing.T) { InstalledAppId: 0, InstalledAppVersionId: 0, AppStoreVersion: 6304, - ValuesOverrideYaml: "## @section Global parameters\n## Global Docker image parameters\n## Please, note that this will override the image parameters, including dependencies, configured to use the global value\n## Current available global Docker image parameters: imageRegistry, imagePullSecrets and storageClass\n\n## @param global.imageRegistry Global Docker image registry\n## @param global.imagePullSecrets Global Docker registry secret names as an array\n## @param global.storageClass Global StorageClass for Persistent Volume(s)\n##\nglobal:\n imageRegistry: \"\"\n ## E.g.\n ## imagePullSecrets:\n ## - myRegistryKeySecretName\n ##\n imagePullSecrets: []\n storageClass: \"\"\n\n## @section Common parameters\n\n## @param kubeVersion Override Kubernetes version\n##\nkubeVersion: \"\"\n## @param nameOverride String to partially override common.names.fullname template (will maintain the release name)\n##\nnameOverride: \"\"\n## @param fullnameOverride String to fully override common.names.fullname template\n##\nfullnameOverride: \"\"\n## @param clusterDomain Kubernetes Cluster Domain\n##\nclusterDomain: cluster.local\n## @param extraDeploy Extra objects to deploy (evaluated as a template)\n##\nextraDeploy: []\n## @param commonLabels Add labels to all the deployed resources\n##\ncommonLabels: {}\n## @param commonAnnotations Add annotations to all the deployed resources\n##\ncommonAnnotations: {}\n## Enable diagnostic mode in the deployment(s)/statefulset(s)\n##\ndiagnosticMode:\n ## @param diagnosticMode.enabled Enable diagnostic mode (all probes will be disabled and the command will be overridden)\n ##\n enabled: false\n ## @param diagnosticMode.command Command to override all containers in the the deployment(s)/statefulset(s)\n ##\n command:\n - sleep\n ## @param diagnosticMode.args Args to override all containers in the the deployment(s)/statefulset(s)\n ##\n args:\n - infinity\n\n## @section Airflow common parameters\n\n## Authentication parameters\n## ref: https://github.com/bitnami/containers/tree/main/bitnami/airflow#environment-variables\n##\nauth:\n ## @param auth.username Username to access web UI\n ##\n username: user\n ## @param auth.password Password to access web UI\n ##\n password: \"\"\n ## @param auth.fernetKey Fernet key to secure connections\n ## ref: https://airflow.readthedocs.io/en/stable/howto/secure-connections.html\n ## ref: https://bcb.github.io/airflow/fernet-key\n ##\n fernetKey: \"\"\n ## @param auth.secretKey Secret key to run your flask app\n ## ref: https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.html#secret-key\n ##\n secretKey: \"\"\n ## @param auth.existingSecret Name of an existing secret to use for Airflow credentials\n ## `auth.password`, `auth.fernetKey`, and `auth.secretKey` will be ignored and picked up from this secret\n ## The secret must contain the keys `airflow-password`, `airflow-fernet-key` and `airflow-secret-key'\n ## The value is evaluated as a template\n ##\n existingSecret: \"\"\n## @param executor Airflow executor. Allowed values: `SequentialExecutor`, `LocalExecutor`, `CeleryExecutor`, `KubernetesExecutor`, `CeleryKubernetesExecutor` and `LocalKubernetesExecutor`\n## ref: http://airflow.apache.org/docs/stable/executor/index.html\n##\nexecutor: CeleryExecutor\n## @param loadExamples Switch to load some Airflow examples\n##\nloadExamples: false\n## @param configuration Specify content for Airflow config file (auto-generated based on other env. vars otherwise)\n## e.g:\n## configuration: |-\n## [core]\n## dags_folder=/opt/bitnami/airflow/dags\n## ...\n##\nconfiguration: \"\"\n## @param existingConfigmap Name of an existing ConfigMap with the Airflow config file\n##\nexistingConfigmap: \"\"\n## Load custom DAGs from a ConfigMap\n## Note: an init container will be used to prepare the DAGs available in the ConfigMap to be consumed by Airflow containers\n##\ndags:\n ## @param dags.existingConfigmap Name of an existing ConfigMap with all the DAGs files you want to load in Airflow\n ##\n existingConfigmap: \"\"\n ## Bitnami Shell image\n ## ref: https://hub.docker.com/r/bitnami/bitnami-shell/tags/\n ## @param dags.image.registry Init container load-dags image registry\n ## @param dags.image.repository Init container load-dags image repository\n ## @param dags.image.tag Init container load-dags image tag (immutable tags are recommended)\n ## @param dags.image.digest Init container load-dags image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param dags.image.pullPolicy Init container load-dags image pull policy\n ## @param dags.image.pullSecrets Init container load-dags image pull secrets\n ##\n image:\n registry: docker.io\n repository: bitnami/bitnami-shell\n tag: 11-debian-11-r50\n digest: \"\"\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n## @param extraEnvVars Add extra environment variables for all the Airflow pods\n##\nextraEnvVars: []\n## @param extraEnvVarsCM ConfigMap with extra environment variables for all the Airflow pods\n##\nextraEnvVarsCM: \"\"\n## @param extraEnvVarsSecret Secret with extra environment variables for all the Airflow pods\n##\nextraEnvVarsSecret: \"\"\n## @param extraEnvVarsSecrets List of secrets with extra environment variables for all the Airflow pods\n##\nextraEnvVarsSecrets: []\n## @param sidecars Add additional sidecar containers to all the Airflow pods\n## Example:\n## sidecars:\n## - name: your-image-name\n## image: your-image\n## imagePullPolicy: Always\n## ports:\n## - name: portname\n## containerPort: 1234\n##\nsidecars: []\n## @param initContainers Add additional init containers to all the Airflow pods\n## Example:\n## initContainers:\n## - name: your-image-name\n## image: your-image\n## imagePullPolicy: Always\n## ports:\n## - name: portname\n## containerPort: 1234\n##\ninitContainers: []\n## @param extraVolumeMounts Optionally specify extra list of additional volumeMounts for all the Airflow pods\n##\nextraVolumeMounts: []\n## @param extraVolumes Optionally specify extra list of additional volumes for the all the Airflow pods\n##\nextraVolumes: []\n\n## @section Airflow web parameters\n\nweb:\n ## Bitnami Airflow image version\n ## ref: https://hub.docker.com/r/bitnami/airflow/tags/\n ## @param web.image.registry Airflow image registry\n ## @param web.image.repository Airflow image repository\n ## @param web.image.tag Airflow image tag (immutable tags are recommended)\n ## @param web.image.digest Airflow image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param web.image.pullPolicy Airflow image pull policy\n ## @param web.image.pullSecrets Airflow image pull secrets\n ## @param web.image.debug Enable image debug mode\n image:\n registry: docker.io\n repository: bitnami/airflow\n tag: 2.4.2-debian-11-r6\n digest: \"\"\n ## Specify a imagePullPolicy\n ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'\n ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images\n ##\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## Set to true if you would like to see extra information on logs\n ##\n debug: false\n ## @param web.baseUrl URL used to access to Airflow web ui\n ##\n baseUrl: \"\"\n ## @param web.existingConfigmap Name of an existing config map containing the Airflow web config file\n ##\n existingConfigmap: \"\"\n ## @param web.command Override default container command (useful when using custom images)\n ##\n command: []\n ## @param web.args Override default container args (useful when using custom images)\n ##\n args: []\n ## @param web.extraEnvVars Array with extra environment variables to add Airflow web pods\n ##\n extraEnvVars: []\n ## @param web.extraEnvVarsCM ConfigMap containing extra environment variables for Airflow web pods\n ##\n extraEnvVarsCM: \"\"\n ## @param web.extraEnvVarsSecret Secret containing extra environment variables (in case of sensitive data) for Airflow web pods\n ##\n extraEnvVarsSecret: \"\"\n ## @param web.extraEnvVarsSecrets List of secrets with extra environment variables for Airflow web pods\n ##\n extraEnvVarsSecrets: []\n ## @param web.containerPorts.http Airflow web HTTP container port\n ##\n containerPorts:\n http: 8080\n ## @param web.replicaCount Number of Airflow web replicas\n ##\n replicaCount: 1\n ## Configure extra options for Airflow web containers' liveness, readiness and startup probes\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes\n ## @param web.livenessProbe.enabled Enable livenessProbe on Airflow web containers\n ## @param web.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe\n ## @param web.livenessProbe.periodSeconds Period seconds for livenessProbe\n ## @param web.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe\n ## @param web.livenessProbe.failureThreshold Failure threshold for livenessProbe\n ## @param web.livenessProbe.successThreshold Success threshold for livenessProbe\n ##\n livenessProbe:\n enabled: true\n initialDelaySeconds: 180\n periodSeconds: 20\n timeoutSeconds: 5\n failureThreshold: 6\n successThreshold: 1\n ## @param web.readinessProbe.enabled Enable readinessProbe on Airflow web containers\n ## @param web.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe\n ## @param web.readinessProbe.periodSeconds Period seconds for readinessProbe\n ## @param web.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe\n ## @param web.readinessProbe.failureThreshold Failure threshold for readinessProbe\n ## @param web.readinessProbe.successThreshold Success threshold for readinessProbe\n ##\n readinessProbe:\n enabled: true\n initialDelaySeconds: 30\n periodSeconds: 10\n timeoutSeconds: 5\n failureThreshold: 6\n successThreshold: 1\n ## @param web.startupProbe.enabled Enable startupProbe on Airflow web containers\n ## @param web.startupProbe.initialDelaySeconds Initial delay seconds for startupProbe\n ## @param web.startupProbe.periodSeconds Period seconds for startupProbe\n ## @param web.startupProbe.timeoutSeconds Timeout seconds for startupProbe\n ## @param web.startupProbe.failureThreshold Failure threshold for startupProbe\n ## @param web.startupProbe.successThreshold Success threshold for startupProbe\n ##\n startupProbe:\n enabled: false\n initialDelaySeconds: 60\n periodSeconds: 10\n timeoutSeconds: 1\n failureThreshold: 15\n successThreshold: 1\n ## @param web.customLivenessProbe Custom livenessProbe that overrides the default one\n ##\n customLivenessProbe: {}\n ## @param web.customReadinessProbe Custom readinessProbe that overrides the default one\n ##\n customReadinessProbe: {}\n ## @param web.customStartupProbe Custom startupProbe that overrides the default one\n ##\n customStartupProbe: {}\n ## Airflow web resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ## @param web.resources.limits The resources limits for the Airflow web containers\n ## @param web.resources.requests The requested resources for the Airflow web containers\n ##\n resources:\n limits: {}\n requests: {}\n ## Configure Airflow web pods Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod\n ## @param web.podSecurityContext.enabled Enabled Airflow web pods' Security Context\n ## @param web.podSecurityContext.fsGroup Set Airflow web pod's Security Context fsGroup\n ##\n podSecurityContext:\n enabled: true\n fsGroup: 1001\n ## Configure Airflow web containers (only main one) Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container\n ## @param web.containerSecurityContext.enabled Enabled Airflow web containers' Security Context\n ## @param web.containerSecurityContext.runAsUser Set Airflow web containers' Security Context runAsUser\n ## @param web.containerSecurityContext.runAsNonRoot Set Airflow web containers' Security Context runAsNonRoot\n ##\n containerSecurityContext:\n enabled: true\n runAsUser: 1001\n runAsNonRoot: true\n ## @param web.lifecycleHooks for the Airflow web container(s) to automate configuration before or after startup\n ##\n lifecycleHooks: {}\n ## @param web.hostAliases Deployment pod host aliases\n ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/\n ##\n hostAliases: []\n ## @param web.podLabels Add extra labels to the Airflow web pods\n ##\n podLabels: {}\n ## @param web.podAnnotations Add extra annotations to the Airflow web pods\n ##\n podAnnotations: {}\n ## @param web.affinity Affinity for Airflow web pods assignment (evaluated as a template)\n ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity\n ## Note: `web.podAffinityPreset`, `web.podAntiAffinityPreset`, and `web.nodeAffinityPreset` will be ignored when it's set\n ##\n affinity: {}\n ## Node affinity preset\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity\n ## @param web.nodeAffinityPreset.key Node label key to match. Ignored if `web.affinity` is set.\n ## @param web.nodeAffinityPreset.type Node affinity preset type. Ignored if `web.affinity` is set. Allowed values: `soft` or `hard`\n ## @param web.nodeAffinityPreset.values Node label values to match. Ignored if `web.affinity` is set.\n ##\n nodeAffinityPreset:\n ## e.g:\n ## key: \"kubernetes.io/e2e-az-name\"\n ##\n key: \"\"\n type: \"\"\n ## e.g:\n ## values:\n ## - e2e-az1\n ## - e2e-az2\n ##\n values: []\n ## @param web.nodeSelector Node labels for Airflow web pods assignment\n ## Ref: https://kubernetes.io/docs/user-guide/node-selection/\n ##\n nodeSelector: {}\n ## @param web.podAffinityPreset Pod affinity preset. Ignored if `web.affinity` is set. Allowed values: `soft` or `hard`.\n ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAffinityPreset: \"\"\n ## @param web.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `web.affinity` is set. Allowed values: `soft` or `hard`.\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAntiAffinityPreset: soft\n ## @param web.tolerations Tolerations for Airflow web pods assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n ##\n tolerations: []\n ## @param web.topologySpreadConstraints Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template\n ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods\n ##\n topologySpreadConstraints: []\n ## @param web.priorityClassName Priority Class Name\n ## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass\n ##\n priorityClassName: \"\"\n ## @param web.schedulerName Use an alternate scheduler, e.g. \"stork\".\n ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/\n ##\n schedulerName: \"\"\n ## @param web.terminationGracePeriodSeconds Seconds Airflow web pod needs to terminate gracefully\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods\n ##\n terminationGracePeriodSeconds: \"\"\n ## @param web.updateStrategy.type Airflow web deployment strategy type\n ## @param web.updateStrategy.rollingUpdate Airflow web deployment rolling update configuration parameters\n ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy\n ##\n updateStrategy:\n type: RollingUpdate\n rollingUpdate: {}\n ## @param web.sidecars Add additional sidecar containers to the Airflow web pods\n ## Example:\n ## sidecars:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n sidecars: []\n ## @param web.initContainers Add additional init containers to the Airflow web pods\n ## Example:\n ## initContainers:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n initContainers: []\n ## @param web.extraVolumeMounts Optionally specify extra list of additional volumeMounts for the Airflow web pods\n ##\n extraVolumeMounts: []\n ## @param web.extraVolumes Optionally specify extra list of additional volumes for the Airflow web pods\n ##\n extraVolumes: []\n ## Airflow web Pod Disruption Budget\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/\n ## @param web.pdb.create Deploy a pdb object for the Airflow web pods\n ## @param web.pdb.minAvailable Maximum number/percentage of unavailable Airflow web replicas\n ## @param web.pdb.maxUnavailable Maximum number/percentage of unavailable Airflow web replicas\n ##\n pdb:\n create: false\n minAvailable: 1\n maxUnavailable: \"\"\n\n## @section Airflow scheduler parameters\n\nscheduler:\n ## Bitnami Airflow Scheduler image version\n ## ref: https://hub.docker.com/r/bitnami/airflow-scheduler/tags/\n ## @param scheduler.image.registry Airflow Scheduler image registry\n ## @param scheduler.image.repository Airflow Scheduler image repository\n ## @param scheduler.image.tag Airflow Scheduler image tag (immutable tags are recommended)\n ## @param scheduler.image.digest Airflow Schefuler image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param scheduler.image.pullPolicy Airflow Scheduler image pull policy\n ## @param scheduler.image.pullSecrets Airflow Scheduler image pull secrets\n ## @param scheduler.image.debug Enable image debug mode\n ##\n image:\n registry: docker.io\n repository: bitnami/airflow-scheduler\n tag: 2.4.2-debian-11-r4\n digest: \"\"\n ## Specify a imagePullPolicy\n ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'\n ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images\n ##\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## Set to true if you would like to see extra information on logs\n ##\n debug: false\n ## @param scheduler.replicaCount Number of scheduler replicas\n ##\n replicaCount: 1\n ## @param scheduler.command Override cmd\n ##\n command: []\n ## @param scheduler.args Override args\n ##\n args: []\n ## @param scheduler.extraEnvVars Add extra environment variables\n ##\n extraEnvVars: []\n ## @param scheduler.extraEnvVarsCM ConfigMap with extra environment variables\n ##\n extraEnvVarsCM: \"\"\n ## @param scheduler.extraEnvVarsSecret Secret with extra environment variables\n ##\n extraEnvVarsSecret: \"\"\n ## @param scheduler.extraEnvVarsSecrets List of secrets with extra environment variables for Airflow scheduler pods\n ##\n extraEnvVarsSecrets: []\n ## @param scheduler.customLivenessProbe Custom livenessProbe that overrides the default one\n ##\n customLivenessProbe: {}\n ## @param scheduler.customReadinessProbe Custom readinessProbe that overrides the default one\n ##\n customReadinessProbe: {}\n ## @param scheduler.customStartupProbe Custom startupProbe that overrides the default one\n ##\n customStartupProbe: {}\n ## Airflow scheduler resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ## @param scheduler.resources.limits The resources limits for the Airflow scheduler containers\n ## @param scheduler.resources.requests The requested resources for the Airflow scheduler containers\n ##\n resources:\n limits: {}\n requests: {}\n ## Configure Airflow scheduler pods Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod\n ## @param scheduler.podSecurityContext.enabled Enabled Airflow scheduler pods' Security Context\n ## @param scheduler.podSecurityContext.fsGroup Set Airflow scheduler pod's Security Context fsGroup\n ##\n podSecurityContext:\n enabled: true\n fsGroup: 1001\n ## Configure Airflow scheduler containers (only main one) Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container\n ## @param scheduler.containerSecurityContext.enabled Enabled Airflow scheduler containers' Security Context\n ## @param scheduler.containerSecurityContext.runAsUser Set Airflow scheduler containers' Security Context runAsUser\n ## @param scheduler.containerSecurityContext.runAsNonRoot Set Airflow scheduler containers' Security Context runAsNonRoot\n ##\n containerSecurityContext:\n enabled: true\n runAsUser: 1001\n runAsNonRoot: true\n ## @param scheduler.lifecycleHooks for the Airflow scheduler container(s) to automate configuration before or after startup\n ##\n lifecycleHooks: {}\n ## @param scheduler.hostAliases Deployment pod host aliases\n ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/\n ##\n hostAliases: []\n ## @param scheduler.podLabels Add extra labels to the Airflow scheduler pods\n ##\n podLabels: {}\n ## @param scheduler.podAnnotations Add extra annotations to the Airflow scheduler pods\n ##\n podAnnotations: {}\n ## @param scheduler.affinity Affinity for Airflow scheduler pods assignment (evaluated as a template)\n ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity\n ## Note: `scheduler.podAffinityPreset`, `scheduler.podAntiAffinityPreset`, and `scheduler.nodeAffinityPreset` will be ignored when it's set\n ##\n affinity: {}\n ## Node affinity preset\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity\n ## @param scheduler.nodeAffinityPreset.key Node label key to match. Ignored if `scheduler.affinity` is set.\n ## @param scheduler.nodeAffinityPreset.type Node affinity preset type. Ignored if `scheduler.affinity` is set. Allowed values: `soft` or `hard`\n ## @param scheduler.nodeAffinityPreset.values Node label values to match. Ignored if `scheduler.affinity` is set.\n ##\n nodeAffinityPreset:\n ## e.g:\n ## key: \"kubernetes.io/e2e-az-name\"\n ##\n key: \"\"\n type: \"\"\n ## e.g:\n ## values:\n ## - e2e-az1\n ## - e2e-az2\n ##\n values: []\n ## @param scheduler.nodeSelector Node labels for Airflow scheduler pods assignment\n ## Ref: https://kubernetes.io/docs/user-guide/node-selection/\n ##\n nodeSelector: {}\n ## @param scheduler.podAffinityPreset Pod affinity preset. Ignored if `scheduler.affinity` is set. Allowed values: `soft` or `hard`.\n ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAffinityPreset: \"\"\n ## @param scheduler.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `scheduler.affinity` is set. Allowed values: `soft` or `hard`.\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAntiAffinityPreset: soft\n ## @param scheduler.tolerations Tolerations for Airflow scheduler pods assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n ##\n tolerations: []\n ## @param scheduler.topologySpreadConstraints Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template\n ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods\n ##\n topologySpreadConstraints: []\n ## @param scheduler.priorityClassName Priority Class Name\n ## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass\n ##\n priorityClassName: \"\"\n ## @param scheduler.schedulerName Use an alternate scheduler, e.g. \"stork\".\n ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/\n ##\n schedulerName: \"\"\n ## @param scheduler.terminationGracePeriodSeconds Seconds Airflow scheduler pod needs to terminate gracefully\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods\n ##\n terminationGracePeriodSeconds: \"\"\n ## @param scheduler.updateStrategy.type Airflow scheduler deployment strategy type\n ## @param scheduler.updateStrategy.rollingUpdate Airflow scheduler deployment rolling update configuration parameters\n ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy\n ##\n updateStrategy:\n type: RollingUpdate\n rollingUpdate: {}\n ## @param scheduler.sidecars Add additional sidecar containers to the Airflow scheduler pods\n ## Example:\n ## sidecars:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n sidecars: []\n ## @param scheduler.initContainers Add additional init containers to the Airflow scheduler pods\n ## Example:\n ## initContainers:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n initContainers: []\n ## @param scheduler.extraVolumeMounts Optionally specify extra list of additional volumeMounts for the Airflow scheduler pods\n ##\n extraVolumeMounts: []\n ## @param scheduler.extraVolumes Optionally specify extra list of additional volumes for the Airflow scheduler pods\n ##\n extraVolumes: []\n ## Airflow scheduler Pod Disruption Budget\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/\n ## @param scheduler.pdb.create Deploy a pdb object for the Airflow scheduler pods\n ## @param scheduler.pdb.minAvailable Maximum number/percentage of unavailable Airflow scheduler replicas\n ## @param scheduler.pdb.maxUnavailable Maximum number/percentage of unavailable Airflow scheduler replicas\n ##\n pdb:\n create: false\n minAvailable: 1\n maxUnavailable: \"\"\n\n## @section Airflow worker parameters\n\nworker:\n ## Bitnami Airflow Worker image version\n ## ref: https://hub.docker.com/r/bitnami/airflow-worker/tags/\n ## @param worker.image.registry Airflow Worker image registry\n ## @param worker.image.repository Airflow Worker image repository\n ## @param worker.image.tag Airflow Worker image tag (immutable tags are recommended)\n ## @param worker.image.digest Airflow Worker image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param worker.image.pullPolicy Airflow Worker image pull policy\n ## @param worker.image.pullSecrets Airflow Worker image pull secrets\n ## @param worker.image.debug Enable image debug mode\n ##\n image:\n registry: docker.io\n repository: bitnami/airflow-worker\n tag: 2.4.2-debian-11-r4\n digest: \"\"\n ## Specify a imagePullPolicy\n ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'\n ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images\n ##\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## Set to true if you would like to see extra information on logs\n ##\n debug: false\n ## @param worker.command Override default container command (useful when using custom images)\n ##\n command: []\n ## @param worker.args Override default container args (useful when using custom images)\n ##\n args: []\n ## @param worker.extraEnvVars Array with extra environment variables to add Airflow worker pods\n ##\n extraEnvVars: []\n ## @param worker.extraEnvVarsCM ConfigMap containing extra environment variables for Airflow worker pods\n ##\n extraEnvVarsCM: \"\"\n ## @param worker.extraEnvVarsSecret Secret containing extra environment variables (in case of sensitive data) for Airflow worker pods\n ##\n extraEnvVarsSecret: \"\"\n ## @param worker.extraEnvVarsSecrets List of secrets with extra environment variables for Airflow worker pods\n ##\n extraEnvVarsSecrets: []\n ## @param worker.containerPorts.http Airflow worker HTTP container port\n ##\n containerPorts:\n http: 8793\n ## @param worker.replicaCount Number of Airflow worker replicas\n ##\n replicaCount: 1\n ## Configure extra options for Airflow worker containers' liveness, readiness and startup probes\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes\n ## @param worker.livenessProbe.enabled Enable livenessProbe on Airflow worker containers\n ## @param worker.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe\n ## @param worker.livenessProbe.periodSeconds Period seconds for livenessProbe\n ## @param worker.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe\n ## @param worker.livenessProbe.failureThreshold Failure threshold for livenessProbe\n ## @param worker.livenessProbe.successThreshold Success threshold for livenessProbe\n ##\n livenessProbe:\n enabled: true\n initialDelaySeconds: 180\n periodSeconds: 20\n timeoutSeconds: 5\n failureThreshold: 6\n successThreshold: 1\n ## @param worker.readinessProbe.enabled Enable readinessProbe on Airflow worker containers\n ## @param worker.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe\n ## @param worker.readinessProbe.periodSeconds Period seconds for readinessProbe\n ## @param worker.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe\n ## @param worker.readinessProbe.failureThreshold Failure threshold for readinessProbe\n ## @param worker.readinessProbe.successThreshold Success threshold for readinessProbe\n ##\n readinessProbe:\n enabled: true\n initialDelaySeconds: 30\n periodSeconds: 10\n timeoutSeconds: 5\n failureThreshold: 6\n successThreshold: 1\n ## @param worker.startupProbe.enabled Enable startupProbe on Airflow worker containers\n ## @param worker.startupProbe.initialDelaySeconds Initial delay seconds for startupProbe\n ## @param worker.startupProbe.periodSeconds Period seconds for startupProbe\n ## @param worker.startupProbe.timeoutSeconds Timeout seconds for startupProbe\n ## @param worker.startupProbe.failureThreshold Failure threshold for startupProbe\n ## @param worker.startupProbe.successThreshold Success threshold for startupProbe\n ##\n startupProbe:\n enabled: false\n initialDelaySeconds: 60\n periodSeconds: 10\n timeoutSeconds: 1\n failureThreshold: 15\n successThreshold: 1\n ## @param worker.customLivenessProbe Custom livenessProbe that overrides the default one\n ##\n customLivenessProbe: {}\n ## @param worker.customReadinessProbe Custom readinessProbe that overrides the default one\n ##\n customReadinessProbe: {}\n ## @param worker.customStartupProbe Custom startupProbe that overrides the default one\n ##\n customStartupProbe: {}\n ## Airflow worker resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ## @param worker.resources.limits The resources limits for the Airflow worker containers\n ## @param worker.resources.requests The requested resources for the Airflow worker containers\n ##\n resources:\n limits: {}\n requests: {}\n ## Configure Airflow worker pods Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod\n ## @param worker.podSecurityContext.enabled Enabled Airflow worker pods' Security Context\n ## @param worker.podSecurityContext.fsGroup Set Airflow worker pod's Security Context fsGroup\n ##\n podSecurityContext:\n enabled: true\n fsGroup: 1001\n ## Configure Airflow worker containers (only main one) Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container\n ## @param worker.containerSecurityContext.enabled Enabled Airflow worker containers' Security Context\n ## @param worker.containerSecurityContext.runAsUser Set Airflow worker containers' Security Context runAsUser\n ## @param worker.containerSecurityContext.runAsNonRoot Set Airflow worker containers' Security Context runAsNonRoot\n ##\n containerSecurityContext:\n enabled: true\n runAsUser: 1001\n runAsNonRoot: true\n ## @param worker.lifecycleHooks for the Airflow worker container(s) to automate configuration before or after startup\n ##\n lifecycleHooks: {}\n ## @param worker.hostAliases Deployment pod host aliases\n ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/\n ##\n hostAliases: []\n ## @param worker.podLabels Add extra labels to the Airflow worker pods\n ##\n podLabels: {}\n ## @param worker.podAnnotations Add extra annotations to the Airflow worker pods\n ##\n podAnnotations: {}\n ## @param worker.affinity Affinity for Airflow worker pods assignment (evaluated as a template)\n ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity\n ## Note: `worker.podAffinityPreset`, `worker.podAntiAffinityPreset`, and `worker.nodeAffinityPreset` will be ignored when it's set\n ##\n affinity: {}\n ## Node affinity preset\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity\n ## @param worker.nodeAffinityPreset.key Node label key to match. Ignored if `worker.affinity` is set.\n ## @param worker.nodeAffinityPreset.type Node affinity preset type. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard`\n ## @param worker.nodeAffinityPreset.values Node label values to match. Ignored if `worker.affinity` is set.\n ##\n nodeAffinityPreset:\n ## e.g:\n ## key: \"kubernetes.io/e2e-az-name\"\n ##\n key: \"\"\n type: \"\"\n ## e.g:\n ## values:\n ## - e2e-az1\n ## - e2e-az2\n ##\n values: []\n ## @param worker.nodeSelector Node labels for Airflow worker pods assignment\n ## Ref: https://kubernetes.io/docs/user-guide/node-selection/\n ##\n nodeSelector: {}\n ## @param worker.podAffinityPreset Pod affinity preset. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard`.\n ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAffinityPreset: \"\"\n ## @param worker.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard`.\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAntiAffinityPreset: soft\n ## @param worker.tolerations Tolerations for Airflow worker pods assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n ##\n tolerations: []\n ## @param worker.topologySpreadConstraints Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template\n ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods\n ##\n topologySpreadConstraints: []\n ## @param worker.priorityClassName Priority Class Name\n ## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass\n ##\n priorityClassName: \"\"\n ## @param worker.schedulerName Use an alternate scheduler, e.g. \"stork\".\n ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/\n ##\n schedulerName: \"\"\n ## @param worker.terminationGracePeriodSeconds Seconds Airflow worker pod needs to terminate gracefully\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods\n ##\n terminationGracePeriodSeconds: \"\"\n ## @param worker.updateStrategy.type Airflow worker deployment strategy type\n ## @param worker.updateStrategy.rollingUpdate Airflow worker deployment rolling update configuration parameters\n ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy\n ##\n updateStrategy:\n type: RollingUpdate\n rollingUpdate: {}\n ## @param worker.sidecars Add additional sidecar containers to the Airflow worker pods\n ## Example:\n ## sidecars:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n sidecars: []\n ## @param worker.initContainers Add additional init containers to the Airflow worker pods\n ## Example:\n ## initContainers:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n initContainers: []\n ## @param worker.extraVolumeMounts Optionally specify extra list of additional volumeMounts for the Airflow worker pods\n ##\n extraVolumeMounts: []\n ## @param worker.extraVolumes Optionally specify extra list of additional volumes for the Airflow worker pods\n ##\n extraVolumes: []\n ## @param worker.extraVolumeClaimTemplates Optionally specify extra list of volumesClaimTemplates for the Airflow worker statefulset\n ##\n extraVolumeClaimTemplates: []\n ## @param worker.podTemplate Template to replace the default one to be use when `executor=KubernetesExecutor` to create Airflow worker pods\n ##\n podTemplate: {}\n ## Airflow worker Pod Disruption Budget\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/\n ## @param worker.pdb.create Deploy a pdb object for the Airflow worker pods\n ## @param worker.pdb.minAvailable Maximum number/percentage of unavailable Airflow worker replicas\n ## @param worker.pdb.maxUnavailable Maximum number/percentage of unavailable Airflow worker replicas\n ##\n pdb:\n create: false\n minAvailable: 1\n maxUnavailable: \"\"\n ## Enable HorizontalPodAutoscaler for worker pods\n ## ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/\n ## @param worker.autoscaling.enabled Whether enable horizontal pod autoscaler\n ## @param worker.autoscaling.minReplicas Configure a minimum amount of pods\n ## @param worker.autoscaling.maxReplicas Configure a maximum amount of pods\n ## @param worker.autoscaling.targetCPU Define the CPU target to trigger the scaling actions (utilization percentage)\n ## @param worker.autoscaling.targetMemory Define the memory target to trigger the scaling actions (utilization percentage)\n ##\n autoscaling:\n enabled: false\n minReplicas: 1\n maxReplicas: 3\n targetCPU: 80\n targetMemory: 80\n\n## @section Airflow git sync parameters\n\n## Configure Git to pull dags and plugins\n##\ngit:\n ## Bitnami Git image version\n ## ref: https://hub.docker.com/r/bitnami/git/tags/\n ## @param git.image.registry Git image registry\n ## @param git.image.repository Git image repository\n ## @param git.image.tag Git image tag (immutable tags are recommended)\n ## @param git.image.digest Git image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param git.image.pullPolicy Git image pull policy\n ## @param git.image.pullSecrets Git image pull secrets\n ##\n image:\n registry: docker.io\n repository: bitnami/git\n tag: 2.38.1-debian-11-r7\n digest: \"\"\n ## Specify a imagePullPolicy\n ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'\n ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images\n ##\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## Get DAG files from git repositories\n ## @param git.dags.enabled Enable in order to download DAG files from git repositories.\n ## @param git.dags.repositories [array] Array of repositories from which to download DAG files\n ##\n dags:\n enabled: false\n ## Name for repositories can be anything unique and must follow same naming conventions as kubernetes.\n ## Kubernetes resources can have names up to 253 characters long. The characters allowed in names are:\n ## digits (0-9), lower case letters (a-z), -, and .\n ## Example:\n ## - repository: https://github.com/myuser/myrepo\n ## branch: main\n ## name: my-dags\n ## path: /\n ##\n repositories:\n - repository: \"\"\n ## Branch from repository to checkout\n ##\n branch: \"\"\n ## An unique identifier for repository, must be unique for each repository\n ##\n name: \"\"\n ## Path to a folder in the repository containing the dags\n ##\n path: \"\"\n ## Get Plugins files from git repositories.\n ## @param git.plugins.enabled Enable in order to download Plugins files from git repositories.\n ## @param git.plugins.repositories [array] Array of repositories from which to download DAG files\n ##\n plugins:\n enabled: false\n repositories:\n - repository: \"\"\n ## Branch from repository to checkout\n ##\n branch: \"\"\n ## An unique identifier for repository, must be unique for each repository\n ##\n name: \"\"\n ## Path to a folder in the repository containing the plugins\n ##\n path: \"\"\n ## Properties for the Clone init container\n ## @param git.clone.command Override cmd\n ## @param git.clone.args Override args\n ## @param git.clone.extraVolumeMounts Add extra volume mounts\n ## @param git.clone.extraEnvVars Add extra environment variables\n ## @param git.clone.extraEnvVarsCM ConfigMap with extra environment variables\n ## @param git.clone.extraEnvVarsSecret Secret with extra environment variables\n ## @param git.clone.resources Clone init container resource requests and limits\n ##\n clone:\n command: []\n args: []\n extraVolumeMounts: []\n extraEnvVars: []\n extraEnvVarsCM: \"\"\n extraEnvVarsSecret: \"\"\n ## Clone init container resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ##\n resources: {}\n ## Properties for the Sync sidecar container\n ## @param git.sync.interval Interval in seconds to pull the git repository containing the plugins and/or DAG files\n ## @param git.sync.command Override cmd\n ## @param git.sync.args Override args\n ## @param git.sync.extraVolumeMounts Add extra volume mounts\n ## @param git.sync.extraEnvVars Add extra environment variables\n ## @param git.sync.extraEnvVarsCM ConfigMap with extra environment variables\n ## @param git.sync.extraEnvVarsSecret Secret with extra environment variables\n ## @param git.sync.resources Sync sidecar container resource requests and limits\n ##\n sync:\n interval: 60\n command: []\n args: []\n extraVolumeMounts: []\n extraEnvVars: []\n extraEnvVarsCM: \"\"\n extraEnvVarsSecret: \"\"\n ## Sync sidecar container resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ##\n resources: {}\n\n## @section Airflow ldap parameters\n\n## LDAP configuration\n## @param ldap.enabled Enable LDAP authentication\n## @param ldap.uri Server URI, eg. ldap://ldap_server:389\n## DEPRECATED ldap.base It will be removed in a future release, please use 'ldap.basedn' instead\n## @param ldap.basedn Base of the search, eg. ou=example,o=org.\n## DEPRECATED ldap.uidField It will be removed in a future release,, please use 'ldap.searchAttribute' instead\n## @param ldap.searchAttribute if doing an indirect bind to ldap, this is the field that matches the username when searching for the account to bind to\n## @param ldap.binddn DN of the account used to search in the LDAP server.\n## @param ldap.bindpw Bind Password\n## @param ldap.userRegistration Set to True to enable user self registration\n## @param ldap.userRegistrationRole Set role name to be assign when a user registers himself. This role must already exist. Mandatory when using ldap.userRegistration\n## @param ldap.rolesMapping mapping from LDAP DN to a list of roles\n## @param ldap.rolesSyncAtLogin replace ALL the user's roles each login, or only on registration\n##\nldap:\n enabled: false\n uri: \"ldap://ldap_server:389\"\n basedn: \"dc=example,dc=org\"\n searchAttribute: \"cn\"\n binddn: \"cn=admin,dc=example,dc=org\"\n bindpw: \"\"\n userRegistration: 'True'\n userRegistrationRole: \"Public\"\n rolesMapping: '{ \"cn=All,ou=Groups,dc=example,dc=org\": [\"User\"], \"cn=Admins,ou=Groups,dc=example,dc=org\": [\"Admin\"], }'\n rolesSyncAtLogin: 'True'\n\n ## SSL/TLS parameters for LDAP\n ## @param ldap.tls.enabled Enabled TLS/SSL for LDAP, you must include the CA file.\n ## @param ldap.tls.allowSelfSigned Allow to use self signed certificates\n ## DEPRECATED ldap.tls.CAcertificateSecret It will be removed in a future release, please use ldap.tls.certificatesSecret instead\n ## @param ldap.tls.certificatesSecret Name of the existing secret containing the certificate CA file that will be used by ldap client\n ## @param ldap.tls.certificatesMountPath Where LDAP certifcates are mounted.\n ## DEPRECATED ldap.tls.CAcertificateFilename It will be removed in a future release, please use ldap.tls.CAFilename instead\n ## @param ldap.tls.CAFilename LDAP CA cert filename\n ##\n tls:\n enabled: false\n allowSelfSigned: true\n certificatesSecret: \"\"\n certificatesMountPath: /opt/bitnami/airflow/conf/certs\n CAFilename: \"\"\n\n## @section Traffic Exposure Parameters\n\n## Airflow service parameters\n##\nservice:\n ## @param service.type Airflow service type\n ##\n type: ClusterIP\n ## @param service.ports.http Airflow service HTTP port\n ##\n ports:\n http: 8080\n ## Node ports to expose\n ## @param service.nodePorts.http Node port for HTTP\n ## NOTE: choose port between <30000-32767>\n ##\n nodePorts:\n http: \"\"\n ## @param service.sessionAffinity Control where client requests go, to the same pod or round-robin\n ## Values: ClientIP or None\n ## ref: https://kubernetes.io/docs/user-guide/services/\n ##\n sessionAffinity: None\n ## @param service.sessionAffinityConfig Additional settings for the sessionAffinity\n ## sessionAffinityConfig:\n ## clientIP:\n ## timeoutSeconds: 300\n ##\n sessionAffinityConfig: {}\n ## @param service.clusterIP Airflow service Cluster IP\n ## e.g.:\n ## clusterIP: None\n ##\n clusterIP: \"\"\n ## @param service.loadBalancerIP Airflow service Load Balancer IP\n ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-loadbalancer\n ##\n loadBalancerIP: \"\"\n ## @param service.loadBalancerSourceRanges Airflow service Load Balancer sources\n ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service\n ## e.g:\n ## loadBalancerSourceRanges:\n ## - 10.10.10.0/24\n ##\n loadBalancerSourceRanges: []\n ## @param service.externalTrafficPolicy Airflow service external traffic policy\n ## ref https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip\n ##\n externalTrafficPolicy: Cluster\n ## @param service.annotations Additional custom annotations for Airflow service\n ##\n annotations: {}\n ## @param service.extraPorts Extra port to expose on Airflow service\n ##\n extraPorts: []\n\n## Airflow ingress parameters\n## ref: https://kubernetes.io/docs/user-guide/ingress/\n##\ningress:\n ## @param ingress.enabled Enable ingress record generation for Airflow\n ##\n enabled: false\n ## @param ingress.ingressClassName IngressClass that will be be used to implement the Ingress (Kubernetes 1.18+)\n ## This is supported in Kubernetes 1.18+ and required if you have more than one IngressClass marked as the default for your cluster .\n ## ref: https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/\n ##\n ingressClassName: \"\"\n ## @param ingress.pathType Ingress path type\n ##\n pathType: ImplementationSpecific\n ## @param ingress.apiVersion Force Ingress API version (automatically detected if not set)\n ##\n apiVersion: \"\"\n ## @param ingress.hostname Default host for the ingress record\n ##\n hostname: airflow.local\n ## @param ingress.path Default path for the ingress record\n ## NOTE: You may need to set this to '/*' in order to use this with ALB ingress controllers\n ##\n path: /\n ## @param ingress.annotations [object] Additional annotations for the Ingress resource. To enable certificate autogeneration, place here your cert-manager annotations.\n ## Use this parameter to set the required annotations for cert-manager, see\n ## ref: https://cert-manager.io/docs/usage/ingress/#supported-annotations\n ## e.g:\n ## annotations:\n ## kubernetes.io/ingress.class: nginx\n ## cert-manager.io/cluster-issuer: cluster-issuer-name\n ##\n annotations: {}\n ## @param ingress.tls Enable TLS configuration for the host defined at `ingress.hostname` parameter\n ## TLS certificates will be retrieved from a TLS secret with name: `{{- printf \"%s-tls\" .Values.ingress.hostname }}`\n ## You can:\n ## - Use the `ingress.secrets` parameter to create this TLS secret\n ## - Rely on cert-manager to create it by setting the corresponding annotations\n ## - Rely on Helm to create self-signed certificates by setting `ingress.selfSigned=true`\n ##\n tls: false\n ## @param ingress.selfSigned Create a TLS secret for this ingress record using self-signed certificates generated by Helm\n ##\n selfSigned: false\n ## @param ingress.extraHosts An array with additional hostname(s) to be covered with the ingress record\n ## e.g:\n ## extraHosts:\n ## - name: airflow.local\n ## path: /\n ##\n extraHosts: []\n ## @param ingress.extraPaths An array with additional arbitrary paths that may need to be added to the ingress under the main host\n ## e.g:\n ## extraPaths:\n ## - path: /*\n ## backend:\n ## serviceName: ssl-redirect\n ## servicePort: use-annotation\n ##\n extraPaths: []\n ## @param ingress.extraTls TLS configuration for additional hostname(s) to be covered with this ingress record\n ## ref: https://kubernetes.io/docs/concepts/services-networking/ingress/#tls\n ## e.g:\n ## extraTls:\n ## - hosts:\n ## - airflow.local\n ## secretName: airflow.local-tls\n ##\n extraTls: []\n ## @param ingress.secrets Custom TLS certificates as secrets\n ## NOTE: 'key' and 'certificate' are expected in PEM format\n ## NOTE: 'name' should line up with a 'secretName' set further up\n ## If it is not set and you're using cert-manager, this is unneeded, as it will create a secret for you with valid certificates\n ## If it is not set and you're NOT using cert-manager either, self-signed certificates will be created valid for 365 days\n ## It is also possible to create and manage the certificates outside of this helm chart\n ## Please see README.md for more information\n ## e.g:\n ## secrets:\n ## - name: airflow.local-tls\n ## key: |-\n ## -----BEGIN RSA PRIVATE KEY-----\n ## ...\n ## -----END RSA PRIVATE KEY-----\n ## certificate: |-\n ## -----BEGIN CERTIFICATE-----\n ## ...\n ## -----END CERTIFICATE-----\n ##\n secrets: []\n ## @param ingress.extraRules Additional rules to be covered with this ingress record\n ## ref: https://kubernetes.io/docs/concepts/services-networking/ingress/#ingress-rules\n ## e.g:\n ## extraRules:\n ## - host: example.local\n ## http:\n ## path: /\n ## backend:\n ## service:\n ## name: example-svc\n ## port:\n ## name: http\n ##\n extraRules: []\n\n## @section Other Parameters\n\n## Service account for Airflow pods to use.\n## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/\n##\nserviceAccount:\n ## @param serviceAccount.create Enable creation of ServiceAccount for Airflow pods\n ##\n create: false\n ## @param serviceAccount.name The name of the ServiceAccount to use.\n ## If not set and create is true, a name is generated using the common.names.fullname template\n ##\n name: \"\"\n ## @param serviceAccount.automountServiceAccountToken Allows auto mount of ServiceAccountToken on the serviceAccount created\n ## Can be set to false if pods using this serviceAccount do not need to use K8s API\n ##\n automountServiceAccountToken: true\n ## @param serviceAccount.annotations Additional custom annotations for the ServiceAccount\n ##\n annotations: {}\n## Role Based Access\n## Ref: https://kubernetes.io/docs/admin/authorization/rbac/\n## @param rbac.create Create Role and RoleBinding\n##\nrbac:\n create: false\n ## @param rbac.rules Custom RBAC rules to set\n ## e.g:\n ## rules:\n ## - apiGroups:\n ## - \"\"\n ## resources:\n ## - pods\n ## verbs:\n ## - get\n ## - list\n ##\n rules: []\n\n## @section Airflow metrics parameters\n\nmetrics:\n ## @param metrics.enabled Whether or not to create a standalone Airflow exporter to expose Airflow metrics\n ##\n enabled: false\n ## Bitnami Airflow exporter image\n ## ref: https://hub.docker.com/r/bitnami/airflow-exporter/tags/\n ## @param metrics.image.registry Airflow exporter image registry\n ## @param metrics.image.repository Airflow exporter image repository\n ## @param metrics.image.tag Airflow exporter image tag (immutable tags are recommended)\n ## @param metrics.image.digest Airflow exporter image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param metrics.image.pullPolicy Airflow exporter image pull policy\n ## @param metrics.image.pullSecrets Airflow exporter image pull secrets\n ##\n image:\n registry: docker.io\n repository: bitnami/airflow-exporter\n tag: 0.20220314.0-debian-11-r57\n digest: \"\"\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## @param metrics.extraEnvVars Array with extra environment variables to add Airflow exporter pods\n ##\n extraEnvVars: []\n ## @param metrics.extraEnvVarsCM ConfigMap containing extra environment variables for Airflow exporter pods\n ##\n extraEnvVarsCM: \"\"\n ## @param metrics.extraEnvVarsSecret Secret containing extra environment variables (in case of sensitive data) for Airflow exporter pods\n ##\n extraEnvVarsSecret: \"\"\n ## @param metrics.containerPorts.http Airflow exporter metrics container port\n ##\n containerPorts:\n http: 9112\n ## Airflow exporter resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ## @param metrics.resources.limits The resources limits for the container\n ## @param metrics.resources.requests The requested resources for the container\n ##\n resources:\n limits: {}\n requests: {}\n ## Airflow exporter pods' Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod\n ## @param metrics.podSecurityContext.enabled Enable security context for the pods\n ## @param metrics.podSecurityContext.fsGroup Set Airflow exporter pod's Security Context fsGroup\n ##\n podSecurityContext:\n enabled: true\n fsGroup: 1001\n ## Airflow exporter containers' Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container\n ## @param metrics.containerSecurityContext.enabled Enable Airflow exporter containers' Security Context\n ## @param metrics.containerSecurityContext.runAsUser Set Airflow exporter containers' Security Context runAsUser\n ## @param metrics.containerSecurityContext.runAsNonRoot Set Airflow exporter containers' Security Context runAsNonRoot\n ## e.g:\n ## containerSecurityContext:\n ## enabled: true\n ## capabilities:\n ## drop: [\"NET_RAW\"]\n ## readOnlyRootFilesystem: true\n ##\n containerSecurityContext:\n enabled: true\n runAsUser: 1001\n runAsNonRoot: true\n ## @param metrics.lifecycleHooks for the Airflow exporter container(s) to automate configuration before or after startup\n ##\n lifecycleHooks: {}\n ## @param metrics.hostAliases Airflow exporter pods host aliases\n ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/\n ##\n hostAliases: []\n ## @param metrics.podLabels Extra labels for Airflow exporter pods\n ## Ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/\n ##\n podLabels: {}\n ## @param metrics.podAnnotations Extra annotations for Airflow exporter pods\n ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/\n ##\n podAnnotations: {}\n ## @param metrics.podAffinityPreset Pod affinity preset. Ignored if `metrics.affinity` is set. Allowed values: `soft` or `hard`\n ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAffinityPreset: \"\"\n ## @param metrics.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `metrics.affinity` is set. Allowed values: `soft` or `hard`\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAntiAffinityPreset: soft\n ## Node metrics.affinity preset\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity\n ##\n nodeAffinityPreset:\n ## @param metrics.nodeAffinityPreset.type Node affinity preset type. Ignored if `metrics.affinity` is set. Allowed values: `soft` or `hard`\n ##\n type: \"\"\n ## @param metrics.nodeAffinityPreset.key Node label key to match Ignored if `metrics.affinity` is set.\n ## E.g.\n ## key: \"kubernetes.io/e2e-az-name\"\n ##\n key: \"\"\n ## @param metrics.nodeAffinityPreset.values Node label values to match. Ignored if `metrics.affinity` is set.\n ## E.g.\n ## values:\n ## - e2e-az1\n ## - e2e-az2\n ##\n values: []\n ## @param metrics.affinity Affinity for pod assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity\n ## Note: metrics.podAffinityPreset, metrics.podAntiAffinityPreset, and metrics.nodeAffinityPreset will be ignored when it's set\n ##\n affinity: {}\n ## @param metrics.nodeSelector Node labels for pod assignment\n ## Ref: https://kubernetes.io/docs/user-guide/node-selection/\n ##\n nodeSelector: {}\n ## @param metrics.tolerations Tolerations for pod assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n ##\n tolerations: []\n ## @param metrics.schedulerName Name of the k8s scheduler (other than default) for Airflow exporter\n ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/\n ##\n schedulerName: \"\"\n ## Airflow exporter service configuration\n ##\n service:\n ## @param metrics.service.ports.http Airflow exporter metrics service port\n ##\n ports:\n http: 9112\n ## @param metrics.service.clusterIP Static clusterIP or None for headless services\n ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#choosing-your-own-ip-address\n ##\n clusterIP: \"\"\n ## @param metrics.service.sessionAffinity Control where client requests go, to the same pod or round-robin\n ## Values: ClientIP or None\n ## ref: https://kubernetes.io/docs/user-guide/services/\n ##\n sessionAffinity: None\n ## @param metrics.service.annotations [object] Annotations for the Airflow exporter service\n ##\n annotations:\n prometheus.io/scrape: \"true\"\n prometheus.io/port: \"{{ .Values.metrics.service.ports.http }}\"\n ## Prometheus Operator ServiceMonitor configuration\n ##\n serviceMonitor:\n ## @param metrics.serviceMonitor.enabled if `true`, creates a Prometheus Operator ServiceMonitor (requires `metrics.enabled` to be `true`)\n ##\n enabled: false\n ## @param metrics.serviceMonitor.namespace Namespace in which Prometheus is running\n ##\n namespace: \"\"\n ## @param metrics.serviceMonitor.interval Interval at which metrics should be scraped\n ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint\n ##\n interval: \"\"\n ## @param metrics.serviceMonitor.scrapeTimeout Timeout after which the scrape is ended\n ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint\n ##\n scrapeTimeout: \"\"\n ## @param metrics.serviceMonitor.labels Additional labels that can be used so ServiceMonitor will be discovered by Prometheus\n ##\n labels: {}\n ## @param metrics.serviceMonitor.selector Prometheus instance selector labels\n ## ref: https://github.com/bitnami/charts/tree/main/bitnami/prometheus-operator#prometheus-configuration\n ##\n selector: {}\n ## @param metrics.serviceMonitor.relabelings RelabelConfigs to apply to samples before scraping\n ##\n relabelings: []\n ## @param metrics.serviceMonitor.metricRelabelings MetricRelabelConfigs to apply to samples before ingestion\n ##\n metricRelabelings: []\n ## @param metrics.serviceMonitor.honorLabels Specify honorLabels parameter to add the scrape endpoint\n ##\n honorLabels: false\n ## @param metrics.serviceMonitor.jobLabel The name of the label on the target service to use as the job name in prometheus.\n ##\n jobLabel: \"\"\n\n## @section Airflow database parameters\n\n## PostgreSQL chart configuration\n## ref: https://github.com/bitnami/charts/blob/main/bitnami/postgresql/values.yaml\n## @param postgresql.enabled Switch to enable or disable the PostgreSQL helm chart\n## @param postgresql.auth.enablePostgresUser Assign a password to the \"postgres\" admin user. Otherwise, git access will be blocked for this user\n## @param postgresql.auth.username Name for a custom user to create\n## @param postgresql.auth.password Password for the custom user to create\n## @param postgresql.auth.database Name for a custom database to create\n## @param postgresql.auth.existingSecret Name of existing secret to use for PostgreSQL credentials\n## @param postgresql.architecture PostgreSQL architecture (`standalone` or `replication`)\n##\npostgresql:\n enabled: true\n auth:\n enablePostgresUser: false\n username: bn_airflow\n password: \"\"\n database: bitnami_airflow\n existingSecret: \"\"\n architecture: standalone\n## External PostgreSQL configuration\n## All of these values are only used when postgresql.enabled is set to false\n## @param externalDatabase.host Database host\n## @param externalDatabase.port Database port number\n## @param externalDatabase.user Non-root username for Airflow\n## @param externalDatabase.password Password for the non-root username for Airflow\n## @param externalDatabase.database Airflow database name\n## @param externalDatabase.existingSecret Name of an existing secret resource containing the database credentials\n## @param externalDatabase.existingSecretPasswordKey Name of an existing secret key containing the database credentials\n##\nexternalDatabase:\n host: localhost\n port: 5432\n user: bn_airflow\n database: bitnami_airflow\n password: \"\"\n existingSecret: \"\"\n existingSecretPasswordKey: \"\"\n\n## Redis® chart configuration\n## ref: https://github.com/bitnami/charts/blob/main/bitnami/redis/values.yaml\n## @param redis.enabled Switch to enable or disable the Redis® helm\n## @param redis.auth.enabled Enable password authentication\n## @param redis.auth.password Redis® password\n## @param redis.auth.existingSecret The name of an existing secret with Redis® credentials\n## @param redis.architecture Redis® architecture. Allowed values: `standalone` or `replication`\n##\nredis:\n enabled: true\n auth:\n enabled: true\n ## Redis® password (both master and slave). Defaults to a random 10-character alphanumeric string if not set and auth.enabled is true.\n ## It should always be set using the password value or in the existingSecret to avoid issues\n ## with Airflow.\n ## The password value is ignored if existingSecret is set\n password: \"\"\n existingSecret: \"\"\n architecture: standalone\n\n## External Redis® configuration\n## All of these values are only used when redis.enabled is set to false\n## @param externalRedis.host Redis® host\n## @param externalRedis.port Redis® port number\n## @param externalRedis.username Redis® username\n## @param externalRedis.password Redis® password\n## @param externalRedis.existingSecret Name of an existing secret resource containing the Redis&trade credentials\n## @param externalRedis.existingSecretPasswordKey Name of an existing secret key containing the Redis&trade credentials\n##\nexternalRedis:\n host: localhost\n port: 6379\n ## Most Redis® implementations do not require a username\n ## to authenticate and it should be enough with the password\n username: \"\"\n password: \"\"\n existingSecret: \"\"\n existingSecretPasswordKey: \"\"\n", + ValuesOverrideYaml: "## @section Global parameters\n## Global Docker image parameters\n## Please, note that this will override the image parameters, including dependencies, configured to use the global value\n## Current available global Docker image parameters: imageRegistry, imagePullSecrets and storageClass\n\n## @param global.imageRegistry Global Docker image registry\n## @param global.imagePullSecrets Global Docker registry secret names as an array\n## @param global.storageClass Global StorageClass for Persistent Volume(s)\n##\nglobal:\n imageRegistry: \"\"\n ## E.g.\n ## imagePullSecrets:\n ## - myRegistryKeySecretName\n ##\n imagePullSecrets: []\n storageClass: \"\"\n\n## @section Common parameters\n\n## @param kubeVersion Override Kubernetes version\n##\nkubeVersion: \"\"\n## @param nameOverride String to partially override common.names.fullname template (will maintain the release name)\n##\nnameOverride: \"\"\n## @param fullnameOverride String to fully override common.names.fullname template\n##\nfullnameOverride: \"\"\n## @param clusterDomain Kubernetes Cluster Domain\n##\nclusterDomain: cluster.local\n## @param extraDeploy Extra objects to deploy (evaluated as a template)\n##\nextraDeploy: []\n## @param commonLabels Add labels to all the deployed resources\n##\ncommonLabels: {}\n## @param commonAnnotations Add annotations to all the deployed resources\n##\ncommonAnnotations: {}\n## Enable diagnostic mode in the deployment(s)/statefulset(s)\n##\ndiagnosticMode:\n ## @param diagnosticMode.enabled Enable diagnostic mode (all probes will be disabled and the command will be overridden)\n ##\n enabled: false\n ## @param diagnosticMode.command Command to override all containers in the the deployment(s)/statefulset(s)\n ##\n command:\n - sleep\n ## @param diagnosticMode.args Args to override all containers in the the deployment(s)/statefulset(s)\n ##\n args:\n - infinity\n\n## @section Airflow common parameters\n\n## Authentication parameters\n## ref: https://github.com/bitnami/containers/tree/main/bitnami/airflow#environment-variables\n##\nauth:\n ## @param auth.username Username to access web UI\n ##\n username: user\n ## @param auth.password Password to access web UI\n ##\n password: \"\"\n ## @param auth.fernetKey Fernet key to secure connections\n ## ref: https://airflow.readthedocs.io/en/stable/howto/secure-connections.html\n ## ref: https://bcb.github.io/airflow/fernet-key\n ##\n fernetKey: \"\"\n ## @param auth.secretKey Secret key to run your flask app\n ## ref: https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.html#secret-key\n ##\n secretKey: \"\"\n ## @param auth.existingSecret Name of an existing secret to use for Airflow credentials\n ## `auth.password`, `auth.fernetKey`, and `auth.secretKey` will be ignored and picked up from this secret\n ## The secret must contain the keys `airflow-password`, `airflow-fernet-key` and `airflow-secret-key'\n ## The value is evaluated as a template\n ##\n existingSecret: \"\"\n## @param executor Airflow executor. Allowed values: `SequentialExecutor`, `LocalExecutor`, `CeleryExecutor`, `KubernetesExecutor`, `CeleryKubernetesExecutor` and `LocalKubernetesExecutor`\n## ref: http://airflow.apache.org/docs/stable/executor/index.html\n##\nexecutor: CeleryExecutor\n## @param loadExamples Switch to load some Airflow examples\n##\nloadExamples: false\n## @param configuration Specify content for Airflow config file (auto-generated based on other env. vars otherwise)\n## e.g:\n## configuration: |-\n## [core]\n## dags_folder=/opt/bitnami/airflow/dags\n## ...\n##\nconfiguration: \"\"\n## @param existingConfigmap Name of an existing ConfigMap with the Airflow config file\n##\nexistingConfigmap: \"\"\n## Load custom DAGs from a ConfigMap\n## Note: an init container will be used to prepare the DAGs available in the ConfigMap to be consumed by Airflow containers\n##\ndags:\n ## @param dags.existingConfigmap Name of an existing ConfigMap with all the DAGs files you want to load in Airflow\n ##\n existingConfigmap: \"\"\n ## Bitnami Shell image\n ## ref: https://hub.docker.com/r/bitnami/bitnami-shell/tags/\n ## @param dags.image.registry Init container load-dags image registry\n ## @param dags.image.repository Init container load-dags image repository\n ## @param dags.image.tag Init container load-dags image tag (immutable tags are recommended)\n ## @param dags.image.digest Init container load-dags image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param dags.image.pullPolicy Init container load-dags image pull policy\n ## @param dags.image.pullSecrets Init container load-dags image pull secrets\n ##\n image:\n registry: docker.io\n repository: bitnami/bitnami-shell\n tag: 11-debian-11-r50\n digest: \"\"\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n## @param extraEnvVars Add extra environment variables for all the Airflow pods\n##\nextraEnvVars: []\n## @param extraEnvVarsCM ConfigMap with extra environment variables for all the Airflow pods\n##\nextraEnvVarsCM: \"\"\n## @param extraEnvVarsSecret Secret with extra environment variables for all the Airflow pods\n##\nextraEnvVarsSecret: \"\"\n## @param extraEnvVarsSecrets List of secrets with extra environment variables for all the Airflow pods\n##\nextraEnvVarsSecrets: []\n## @param sidecars Add additional sidecar containers to all the Airflow pods\n## Example:\n## sidecars:\n## - name: your-image-name\n## image: your-image\n## imagePullPolicy: Always\n## ports:\n## - name: portname\n## containerPort: 1234\n##\nsidecars: []\n## @param initContainers Add additional init containers to all the Airflow pods\n## Example:\n## initContainers:\n## - name: your-image-name\n## image: your-image\n## imagePullPolicy: Always\n## ports:\n## - name: portname\n## containerPort: 1234\n##\ninitContainers: []\n## @param extraVolumeMounts Optionally specify extra list of additional volumeMounts for all the Airflow pods\n##\nextraVolumeMounts: []\n## @param extraVolumes Optionally specify extra list of additional volumes for the all the Airflow pods\n##\nextraVolumes: []\n\n## @section Airflow web parameters\n\nweb:\n ## Bitnami Airflow image version\n ## ref: https://hub.docker.com/r/bitnami/airflow/tags/\n ## @param web.image.registry Airflow image registry\n ## @param web.image.repository Airflow image repository\n ## @param web.image.tag Airflow image tag (immutable tags are recommended)\n ## @param web.image.digest Airflow image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param web.image.pullPolicy Airflow image pull policy\n ## @param web.image.pullSecrets Airflow image pull secrets\n ## @param web.image.debug Enable image debug mode\n image:\n registry: docker.io\n repository: bitnami/airflow\n tag: 2.4.2-debian-11-r6\n digest: \"\"\n ## Specify a imagePullPolicy\n ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'\n ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images\n ##\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## Set to true if you would like to see extra information on logs\n ##\n debug: false\n ## @param web.baseUrl URL used to access to Airflow web ui\n ##\n baseUrl: \"\"\n ## @param web.existingConfigmap Name of an existing config map containing the Airflow web config file\n ##\n existingConfigmap: \"\"\n ## @param web.command Override default container command (useful when using custom images)\n ##\n command: []\n ## @param web.args Override default container args (useful when using custom images)\n ##\n args: []\n ## @param web.extraEnvVars Array with extra environment variables to add Airflow web pods\n ##\n extraEnvVars: []\n ## @param web.extraEnvVarsCM ConfigMap containing extra environment variables for Airflow web pods\n ##\n extraEnvVarsCM: \"\"\n ## @param web.extraEnvVarsSecret Secret containing extra environment variables (in case of sensitive data) for Airflow web pods\n ##\n extraEnvVarsSecret: \"\"\n ## @param web.extraEnvVarsSecrets List of secrets with extra environment variables for Airflow web pods\n ##\n extraEnvVarsSecrets: []\n ## @param web.containerPorts.http Airflow web HTTP container port\n ##\n containerPorts:\n http: 8080\n ## @param web.replicaCount Number of Airflow web replicas\n ##\n replicaCount: 1\n ## Configure extra options for Airflow web containers' liveness, readiness and startup probes\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes\n ## @param web.livenessProbe.enabled Enable livenessProbe on Airflow web containers\n ## @param web.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe\n ## @param web.livenessProbe.periodSeconds Period seconds for livenessProbe\n ## @param web.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe\n ## @param web.livenessProbe.failureThreshold Failure threshold for livenessProbe\n ## @param web.livenessProbe.successThreshold Success threshold for livenessProbe\n ##\n livenessProbe:\n enabled: true\n initialDelaySeconds: 180\n periodSeconds: 20\n timeoutSeconds: 5\n failureThreshold: 6\n successThreshold: 1\n ## @param web.readinessProbe.enabled Enable readinessProbe on Airflow web containers\n ## @param web.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe\n ## @param web.readinessProbe.periodSeconds Period seconds for readinessProbe\n ## @param web.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe\n ## @param web.readinessProbe.failureThreshold Failure threshold for readinessProbe\n ## @param web.readinessProbe.successThreshold Success threshold for readinessProbe\n ##\n readinessProbe:\n enabled: true\n initialDelaySeconds: 30\n periodSeconds: 10\n timeoutSeconds: 5\n failureThreshold: 6\n successThreshold: 1\n ## @param web.startupProbe.enabled Enable startupProbe on Airflow web containers\n ## @param web.startupProbe.initialDelaySeconds Initial delay seconds for startupProbe\n ## @param web.startupProbe.periodSeconds Period seconds for startupProbe\n ## @param web.startupProbe.timeoutSeconds Timeout seconds for startupProbe\n ## @param web.startupProbe.failureThreshold Failure threshold for startupProbe\n ## @param web.startupProbe.successThreshold Success threshold for startupProbe\n ##\n startupProbe:\n enabled: false\n initialDelaySeconds: 60\n periodSeconds: 10\n timeoutSeconds: 1\n failureThreshold: 15\n successThreshold: 1\n ## @param web.customLivenessProbe Custom livenessProbe that overrides the default one\n ##\n customLivenessProbe: {}\n ## @param web.customReadinessProbe Custom readinessProbe that overrides the default one\n ##\n customReadinessProbe: {}\n ## @param web.customStartupProbe Custom startupProbe that overrides the default one\n ##\n customStartupProbe: {}\n ## Airflow web resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ## @param web.resources.limits The resources limits for the Airflow web containers\n ## @param web.resources.requests The requested resources for the Airflow web containers\n ##\n resources:\n limits: {}\n requests: {}\n ## Configure Airflow web pods Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod\n ## @param web.podSecurityContext.enabled Enabled Airflow web pods' Security Context\n ## @param web.podSecurityContext.fsGroup Set Airflow web pod's Security Context fsGroup\n ##\n podSecurityContext:\n enabled: true\n fsGroup: 1001\n ## Configure Airflow web containers (only main one) Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container\n ## @param web.containerSecurityContext.enabled Enabled Airflow web containers' Security Context\n ## @param web.containerSecurityContext.runAsUser Set Airflow web containers' Security Context runAsUser\n ## @param web.containerSecurityContext.runAsNonRoot Set Airflow web containers' Security Context runAsNonRoot\n ##\n containerSecurityContext:\n enabled: true\n runAsUser: 1001\n runAsNonRoot: true\n ## @param web.lifecycleHooks for the Airflow web container(s) to automate configuration before or after startup\n ##\n lifecycleHooks: {}\n ## @param web.hostAliases Deployment pod host aliases\n ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/\n ##\n hostAliases: []\n ## @param web.podLabels Add extra labels to the Airflow web pods\n ##\n podLabels: {}\n ## @param web.podAnnotations Add extra annotations to the Airflow web pods\n ##\n podAnnotations: {}\n ## @param web.affinity Affinity for Airflow web pods assignment (evaluated as a template)\n ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity\n ## Note: `web.podAffinityPreset`, `web.podAntiAffinityPreset`, and `web.nodeAffinityPreset` will be ignored when it's set\n ##\n affinity: {}\n ## Node affinity preset\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity\n ## @param web.nodeAffinityPreset.key Node label key to match. Ignored if `web.affinity` is set.\n ## @param web.nodeAffinityPreset.type Node affinity preset type. Ignored if `web.affinity` is set. Allowed values: `soft` or `hard`\n ## @param web.nodeAffinityPreset.values Node label values to match. Ignored if `web.affinity` is set.\n ##\n nodeAffinityPreset:\n ## e.g:\n ## key: \"kubernetes.io/e2e-az-name\"\n ##\n key: \"\"\n type: \"\"\n ## e.g:\n ## values:\n ## - e2e-az1\n ## - e2e-az2\n ##\n values: []\n ## @param web.nodeSelector Node labels for Airflow web pods assignment\n ## Ref: https://kubernetes.io/docs/user-guide/node-selection/\n ##\n nodeSelector: {}\n ## @param web.podAffinityPreset Pod affinity preset. Ignored if `web.affinity` is set. Allowed values: `soft` or `hard`.\n ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAffinityPreset: \"\"\n ## @param web.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `web.affinity` is set. Allowed values: `soft` or `hard`.\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAntiAffinityPreset: soft\n ## @param web.tolerations Tolerations for Airflow web pods assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n ##\n tolerations: []\n ## @param web.topologySpreadConstraints Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template\n ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods\n ##\n topologySpreadConstraints: []\n ## @param web.priorityClassName Priority Class Name\n ## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass\n ##\n priorityClassName: \"\"\n ## @param web.schedulerName Use an alternate scheduler, e.g. \"stork\".\n ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/\n ##\n schedulerName: \"\"\n ## @param web.terminationGracePeriodSeconds Seconds Airflow web pod needs to terminate gracefully\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods\n ##\n terminationGracePeriodSeconds: \"\"\n ## @param web.updateStrategy.type Airflow web deployment strategy type\n ## @param web.updateStrategy.rollingUpdate Airflow web deployment rolling update configuration parameters\n ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy\n ##\n updateStrategy:\n type: RollingUpdate\n rollingUpdate: {}\n ## @param web.sidecars Add additional sidecar containers to the Airflow web pods\n ## Example:\n ## sidecars:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n sidecars: []\n ## @param web.initContainers Add additional init containers to the Airflow web pods\n ## Example:\n ## initContainers:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n initContainers: []\n ## @param web.extraVolumeMounts Optionally specify extra list of additional volumeMounts for the Airflow web pods\n ##\n extraVolumeMounts: []\n ## @param web.extraVolumes Optionally specify extra list of additional volumes for the Airflow web pods\n ##\n extraVolumes: []\n ## Airflow web Pod Disruption Budget\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/\n ## @param web.pdb.create Deploy a pdb object for the Airflow web pods\n ## @param web.pdb.minAvailable Maximum number/percentage of unavailable Airflow web replicas\n ## @param web.pdb.maxUnavailable Maximum number/percentage of unavailable Airflow web replicas\n ##\n pdb:\n create: false\n minAvailable: 1\n maxUnavailable: \"\"\n\n## @section Airflow scheduler parameters\n\nscheduler:\n ## Bitnami Airflow Scheduler image version\n ## ref: https://hub.docker.com/r/bitnami/airflow-scheduler/tags/\n ## @param scheduler.image.registry Airflow Scheduler image registry\n ## @param scheduler.image.repository Airflow Scheduler image repository\n ## @param scheduler.image.tag Airflow Scheduler image tag (immutable tags are recommended)\n ## @param scheduler.image.digest Airflow Schefuler image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param scheduler.image.pullPolicy Airflow Scheduler image pull policy\n ## @param scheduler.image.pullSecrets Airflow Scheduler image pull secrets\n ## @param scheduler.image.debug Enable image debug mode\n ##\n image:\n registry: docker.io\n repository: bitnami/airflow-scheduler\n tag: 2.4.2-debian-11-r4\n digest: \"\"\n ## Specify a imagePullPolicy\n ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'\n ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images\n ##\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## Set to true if you would like to see extra information on logs\n ##\n debug: false\n ## @param scheduler.replicaCount Number of scheduler replicas\n ##\n replicaCount: 1\n ## @param scheduler.command Override cmd\n ##\n command: []\n ## @param scheduler.args Override args\n ##\n args: []\n ## @param scheduler.extraEnvVars Add extra environment variables\n ##\n extraEnvVars: []\n ## @param scheduler.extraEnvVarsCM ConfigMap with extra environment variables\n ##\n extraEnvVarsCM: \"\"\n ## @param scheduler.extraEnvVarsSecret Secret with extra environment variables\n ##\n extraEnvVarsSecret: \"\"\n ## @param scheduler.extraEnvVarsSecrets List of secrets with extra environment variables for Airflow scheduler pods\n ##\n extraEnvVarsSecrets: []\n ## @param scheduler.customLivenessProbe Custom livenessProbe that overrides the default one\n ##\n customLivenessProbe: {}\n ## @param scheduler.customReadinessProbe Custom readinessProbe that overrides the default one\n ##\n customReadinessProbe: {}\n ## @param scheduler.customStartupProbe Custom startupProbe that overrides the default one\n ##\n customStartupProbe: {}\n ## Airflow scheduler resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ## @param scheduler.resources.limits The resources limits for the Airflow scheduler containers\n ## @param scheduler.resources.requests The requested resources for the Airflow scheduler containers\n ##\n resources:\n limits: {}\n requests: {}\n ## Configure Airflow scheduler pods Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod\n ## @param scheduler.podSecurityContext.enabled Enabled Airflow scheduler pods' Security Context\n ## @param scheduler.podSecurityContext.fsGroup Set Airflow scheduler pod's Security Context fsGroup\n ##\n podSecurityContext:\n enabled: true\n fsGroup: 1001\n ## Configure Airflow scheduler containers (only main one) Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container\n ## @param scheduler.containerSecurityContext.enabled Enabled Airflow scheduler containers' Security Context\n ## @param scheduler.containerSecurityContext.runAsUser Set Airflow scheduler containers' Security Context runAsUser\n ## @param scheduler.containerSecurityContext.runAsNonRoot Set Airflow scheduler containers' Security Context runAsNonRoot\n ##\n containerSecurityContext:\n enabled: true\n runAsUser: 1001\n runAsNonRoot: true\n ## @param scheduler.lifecycleHooks for the Airflow scheduler container(s) to automate configuration before or after startup\n ##\n lifecycleHooks: {}\n ## @param scheduler.hostAliases Deployment pod host aliases\n ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/\n ##\n hostAliases: []\n ## @param scheduler.podLabels Add extra labels to the Airflow scheduler pods\n ##\n podLabels: {}\n ## @param scheduler.podAnnotations Add extra annotations to the Airflow scheduler pods\n ##\n podAnnotations: {}\n ## @param scheduler.affinity Affinity for Airflow scheduler pods assignment (evaluated as a template)\n ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity\n ## Note: `scheduler.podAffinityPreset`, `scheduler.podAntiAffinityPreset`, and `scheduler.nodeAffinityPreset` will be ignored when it's set\n ##\n affinity: {}\n ## Node affinity preset\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity\n ## @param scheduler.nodeAffinityPreset.key Node label key to match. Ignored if `scheduler.affinity` is set.\n ## @param scheduler.nodeAffinityPreset.type Node affinity preset type. Ignored if `scheduler.affinity` is set. Allowed values: `soft` or `hard`\n ## @param scheduler.nodeAffinityPreset.values Node label values to match. Ignored if `scheduler.affinity` is set.\n ##\n nodeAffinityPreset:\n ## e.g:\n ## key: \"kubernetes.io/e2e-az-name\"\n ##\n key: \"\"\n type: \"\"\n ## e.g:\n ## values:\n ## - e2e-az1\n ## - e2e-az2\n ##\n values: []\n ## @param scheduler.nodeSelector Node labels for Airflow scheduler pods assignment\n ## Ref: https://kubernetes.io/docs/user-guide/node-selection/\n ##\n nodeSelector: {}\n ## @param scheduler.podAffinityPreset Pod affinity preset. Ignored if `scheduler.affinity` is set. Allowed values: `soft` or `hard`.\n ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAffinityPreset: \"\"\n ## @param scheduler.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `scheduler.affinity` is set. Allowed values: `soft` or `hard`.\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAntiAffinityPreset: soft\n ## @param scheduler.tolerations Tolerations for Airflow scheduler pods assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n ##\n tolerations: []\n ## @param scheduler.topologySpreadConstraints Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template\n ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods\n ##\n topologySpreadConstraints: []\n ## @param scheduler.priorityClassName Priority Class Name\n ## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass\n ##\n priorityClassName: \"\"\n ## @param scheduler.schedulerName Use an alternate scheduler, e.g. \"stork\".\n ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/\n ##\n schedulerName: \"\"\n ## @param scheduler.terminationGracePeriodSeconds Seconds Airflow scheduler pod needs to terminate gracefully\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods\n ##\n terminationGracePeriodSeconds: \"\"\n ## @param scheduler.updateStrategy.type Airflow scheduler deployment strategy type\n ## @param scheduler.updateStrategy.rollingUpdate Airflow scheduler deployment rolling update configuration parameters\n ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy\n ##\n updateStrategy:\n type: RollingUpdate\n rollingUpdate: {}\n ## @param scheduler.sidecars Add additional sidecar containers to the Airflow scheduler pods\n ## Example:\n ## sidecars:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n sidecars: []\n ## @param scheduler.initContainers Add additional init containers to the Airflow scheduler pods\n ## Example:\n ## initContainers:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n initContainers: []\n ## @param scheduler.extraVolumeMounts Optionally specify extra list of additional volumeMounts for the Airflow scheduler pods\n ##\n extraVolumeMounts: []\n ## @param scheduler.extraVolumes Optionally specify extra list of additional volumes for the Airflow scheduler pods\n ##\n extraVolumes: []\n ## Airflow scheduler Pod Disruption Budget\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/\n ## @param scheduler.pdb.create Deploy a pdb object for the Airflow scheduler pods\n ## @param scheduler.pdb.minAvailable Maximum number/percentage of unavailable Airflow scheduler replicas\n ## @param scheduler.pdb.maxUnavailable Maximum number/percentage of unavailable Airflow scheduler replicas\n ##\n pdb:\n create: false\n minAvailable: 1\n maxUnavailable: \"\"\n\n## @section Airflow worker parameters\n\nworker:\n ## Bitnami Airflow Worker image version\n ## ref: https://hub.docker.com/r/bitnami/airflow-worker/tags/\n ## @param worker.image.registry Airflow Worker image registry\n ## @param worker.image.repository Airflow Worker image repository\n ## @param worker.image.tag Airflow Worker image tag (immutable tags are recommended)\n ## @param worker.image.digest Airflow Worker image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param worker.image.pullPolicy Airflow Worker image pull policy\n ## @param worker.image.pullSecrets Airflow Worker image pull secrets\n ## @param worker.image.debug Enable image debug mode\n ##\n image:\n registry: docker.io\n repository: bitnami/airflow-worker\n tag: 2.4.2-debian-11-r4\n digest: \"\"\n ## Specify a imagePullPolicy\n ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'\n ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images\n ##\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## Set to true if you would like to see extra information on logs\n ##\n debug: false\n ## @param worker.command Override default container command (useful when using custom images)\n ##\n command: []\n ## @param worker.args Override default container args (useful when using custom images)\n ##\n args: []\n ## @param worker.extraEnvVars Array with extra environment variables to add Airflow worker pods\n ##\n extraEnvVars: []\n ## @param worker.extraEnvVarsCM ConfigMap containing extra environment variables for Airflow worker pods\n ##\n extraEnvVarsCM: \"\"\n ## @param worker.extraEnvVarsSecret Secret containing extra environment variables (in case of sensitive data) for Airflow worker pods\n ##\n extraEnvVarsSecret: \"\"\n ## @param worker.extraEnvVarsSecrets List of secrets with extra environment variables for Airflow worker pods\n ##\n extraEnvVarsSecrets: []\n ## @param worker.containerPorts.http Airflow worker HTTP container port\n ##\n containerPorts:\n http: 8793\n ## @param worker.replicaCount Number of Airflow worker replicas\n ##\n replicaCount: 1\n ## Configure extra options for Airflow worker containers' liveness, readiness and startup probes\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes\n ## @param worker.livenessProbe.enabled Enable livenessProbe on Airflow worker containers\n ## @param worker.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe\n ## @param worker.livenessProbe.periodSeconds Period seconds for livenessProbe\n ## @param worker.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe\n ## @param worker.livenessProbe.failureThreshold Failure threshold for livenessProbe\n ## @param worker.livenessProbe.successThreshold Success threshold for livenessProbe\n ##\n livenessProbe:\n enabled: true\n initialDelaySeconds: 180\n periodSeconds: 20\n timeoutSeconds: 5\n failureThreshold: 6\n successThreshold: 1\n ## @param worker.readinessProbe.enabled Enable readinessProbe on Airflow worker containers\n ## @param worker.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe\n ## @param worker.readinessProbe.periodSeconds Period seconds for readinessProbe\n ## @param worker.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe\n ## @param worker.readinessProbe.failureThreshold Failure threshold for readinessProbe\n ## @param worker.readinessProbe.successThreshold Success threshold for readinessProbe\n ##\n readinessProbe:\n enabled: true\n initialDelaySeconds: 30\n periodSeconds: 10\n timeoutSeconds: 5\n failureThreshold: 6\n successThreshold: 1\n ## @param worker.startupProbe.enabled Enable startupProbe on Airflow worker containers\n ## @param worker.startupProbe.initialDelaySeconds Initial delay seconds for startupProbe\n ## @param worker.startupProbe.periodSeconds Period seconds for startupProbe\n ## @param worker.startupProbe.timeoutSeconds Timeout seconds for startupProbe\n ## @param worker.startupProbe.failureThreshold Failure threshold for startupProbe\n ## @param worker.startupProbe.successThreshold Success threshold for startupProbe\n ##\n startupProbe:\n enabled: false\n initialDelaySeconds: 60\n periodSeconds: 10\n timeoutSeconds: 1\n failureThreshold: 15\n successThreshold: 1\n ## @param worker.customLivenessProbe Custom livenessProbe that overrides the default one\n ##\n customLivenessProbe: {}\n ## @param worker.customReadinessProbe Custom readinessProbe that overrides the default one\n ##\n customReadinessProbe: {}\n ## @param worker.customStartupProbe Custom startupProbe that overrides the default one\n ##\n customStartupProbe: {}\n ## Airflow worker resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ## @param worker.resources.limits The resources limits for the Airflow worker containers\n ## @param worker.resources.requests The requested resources for the Airflow worker containers\n ##\n resources:\n limits: {}\n requests: {}\n ## Configure Airflow worker pods Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod\n ## @param worker.podSecurityContext.enabled Enabled Airflow worker pods' Security Context\n ## @param worker.podSecurityContext.fsGroup Set Airflow worker pod's Security Context fsGroup\n ##\n podSecurityContext:\n enabled: true\n fsGroup: 1001\n ## Configure Airflow worker containers (only main one) Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container\n ## @param worker.containerSecurityContext.enabled Enabled Airflow worker containers' Security Context\n ## @param worker.containerSecurityContext.runAsUser Set Airflow worker containers' Security Context runAsUser\n ## @param worker.containerSecurityContext.runAsNonRoot Set Airflow worker containers' Security Context runAsNonRoot\n ##\n containerSecurityContext:\n enabled: true\n runAsUser: 1001\n runAsNonRoot: true\n ## @param worker.lifecycleHooks for the Airflow worker container(s) to automate configuration before or after startup\n ##\n lifecycleHooks: {}\n ## @param worker.hostAliases Deployment pod host aliases\n ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/\n ##\n hostAliases: []\n ## @param worker.podLabels Add extra labels to the Airflow worker pods\n ##\n podLabels: {}\n ## @param worker.podAnnotations Add extra annotations to the Airflow worker pods\n ##\n podAnnotations: {}\n ## @param worker.affinity Affinity for Airflow worker pods assignment (evaluated as a template)\n ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity\n ## Note: `worker.podAffinityPreset`, `worker.podAntiAffinityPreset`, and `worker.nodeAffinityPreset` will be ignored when it's set\n ##\n affinity: {}\n ## Node affinity preset\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity\n ## @param worker.nodeAffinityPreset.key Node label key to match. Ignored if `worker.affinity` is set.\n ## @param worker.nodeAffinityPreset.type Node affinity preset type. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard`\n ## @param worker.nodeAffinityPreset.values Node label values to match. Ignored if `worker.affinity` is set.\n ##\n nodeAffinityPreset:\n ## e.g:\n ## key: \"kubernetes.io/e2e-az-name\"\n ##\n key: \"\"\n type: \"\"\n ## e.g:\n ## values:\n ## - e2e-az1\n ## - e2e-az2\n ##\n values: []\n ## @param worker.nodeSelector Node labels for Airflow worker pods assignment\n ## Ref: https://kubernetes.io/docs/user-guide/node-selection/\n ##\n nodeSelector: {}\n ## @param worker.podAffinityPreset Pod affinity preset. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard`.\n ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAffinityPreset: \"\"\n ## @param worker.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard`.\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAntiAffinityPreset: soft\n ## @param worker.tolerations Tolerations for Airflow worker pods assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n ##\n tolerations: []\n ## @param worker.topologySpreadConstraints Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template\n ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods\n ##\n topologySpreadConstraints: []\n ## @param worker.priorityClassName Priority Class Name\n ## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass\n ##\n priorityClassName: \"\"\n ## @param worker.schedulerName Use an alternate scheduler, e.g. \"stork\".\n ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/\n ##\n schedulerName: \"\"\n ## @param worker.terminationGracePeriodSeconds Seconds Airflow worker pod needs to terminate gracefully\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods\n ##\n terminationGracePeriodSeconds: \"\"\n ## @param worker.updateStrategy.type Airflow worker deployment strategy type\n ## @param worker.updateStrategy.rollingUpdate Airflow worker deployment rolling update configuration parameters\n ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy\n ##\n updateStrategy:\n type: RollingUpdate\n rollingUpdate: {}\n ## @param worker.sidecars Add additional sidecar containers to the Airflow worker pods\n ## Example:\n ## sidecars:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n sidecars: []\n ## @param worker.initContainers Add additional init containers to the Airflow worker pods\n ## Example:\n ## initContainers:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n initContainers: []\n ## @param worker.extraVolumeMounts Optionally specify extra list of additional volumeMounts for the Airflow worker pods\n ##\n extraVolumeMounts: []\n ## @param worker.extraVolumes Optionally specify extra list of additional volumes for the Airflow worker pods\n ##\n extraVolumes: []\n ## @param worker.extraVolumeClaimTemplates Optionally specify extra list of volumesClaimTemplates for the Airflow worker statefulset\n ##\n extraVolumeClaimTemplates: []\n ## @param worker.podTemplate Template to replace the default one to be use when `executor=KubernetesExecutor` to create Airflow worker pods\n ##\n podTemplate: {}\n ## Airflow worker Pod Disruption Budget\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/\n ## @param worker.pdb.create Deploy a pdb object for the Airflow worker pods\n ## @param worker.pdb.minAvailable Maximum number/percentage of unavailable Airflow worker replicas\n ## @param worker.pdb.maxUnavailable Maximum number/percentage of unavailable Airflow worker replicas\n ##\n pdb:\n create: false\n minAvailable: 1\n maxUnavailable: \"\"\n ## Enable HorizontalPodAutoscaler for worker pods\n ## ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/\n ## @param worker.autoscaling.enabled Whether enable horizontal pod autoscaler\n ## @param worker.autoscaling.minReplicas Configure a minimum amount of pods\n ## @param worker.autoscaling.maxReplicas Configure a maximum amount of pods\n ## @param worker.autoscaling.targetCPU Define the CPU target to trigger the scaling actions (utilization percentage)\n ## @param worker.autoscaling.targetMemory Define the memory target to trigger the scaling actions (utilization percentage)\n ##\n autoscaling:\n enabled: false\n minReplicas: 1\n maxReplicas: 3\n targetCPU: 80\n targetMemory: 80\n\n## @section Airflow git sync parameters\n\n## Configure Git to pull dags and plugins\n##\ngit:\n ## Bitnami Git image version\n ## ref: https://hub.docker.com/r/bitnami/git/tags/\n ## @param git.image.registry Git image registry\n ## @param git.image.repository Git image repository\n ## @param git.image.tag Git image tag (immutable tags are recommended)\n ## @param git.image.digest Git image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param git.image.pullPolicy Git image pull policy\n ## @param git.image.pullSecrets Git image pull secrets\n ##\n image:\n registry: docker.io\n repository: bitnami/git\n tag: 2.38.1-debian-11-r7\n digest: \"\"\n ## Specify a imagePullPolicy\n ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'\n ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images\n ##\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## Get DAG files from git repositories\n ## @param git.dags.enabled Enable in order to download DAG files from git repositories.\n ## @param git.dags.repositories [array] Array of repositories from which to download DAG files\n ##\n dags:\n enabled: false\n ## Name for repositories can be anything unique and must follow same naming conventions as kubernetes.\n ## Kubernetes resources can have names up to 253 characters long. The characters allowed in names are:\n ## digits (0-9), lower case letters (a-z), -, and .\n ## Example:\n ## - repository: https://github.com/myuser/myrepo\n ## branch: main\n ## name: my-dags\n ## path: /\n ##\n repositories:\n - repository: \"\"\n ## Branch from repository to checkout\n ##\n branch: \"\"\n ## An unique identifier for repository, must be unique for each repository\n ##\n name: \"\"\n ## Path to a folder in the repository containing the dags\n ##\n path: \"\"\n ## Get Plugins files from git repositories.\n ## @param git.plugins.enabled Enable in order to download Plugins files from git repositories.\n ## @param git.plugins.repositories [array] Array of repositories from which to download DAG files\n ##\n plugins:\n enabled: false\n repositories:\n - repository: \"\"\n ## Branch from repository to checkout\n ##\n branch: \"\"\n ## An unique identifier for repository, must be unique for each repository\n ##\n name: \"\"\n ## Path to a folder in the repository containing the plugins\n ##\n path: \"\"\n ## Properties for the Clone init container\n ## @param git.clone.command Override cmd\n ## @param git.clone.args Override args\n ## @param git.clone.extraVolumeMounts Add extra volume mounts\n ## @param git.clone.extraEnvVars Add extra environment variables\n ## @param git.clone.extraEnvVarsCM ConfigMap with extra environment variables\n ## @param git.clone.extraEnvVarsSecret Secret with extra environment variables\n ## @param git.clone.resources Clone init container resource requests and limits\n ##\n clone:\n command: []\n args: []\n extraVolumeMounts: []\n extraEnvVars: []\n extraEnvVarsCM: \"\"\n extraEnvVarsSecret: \"\"\n ## Clone init container resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ##\n resources: {}\n ## Properties for the Sync sidecar container\n ## @param git.sync.interval Interval in seconds to pull the git repository containing the plugins and/or DAG files\n ## @param git.sync.command Override cmd\n ## @param git.sync.args Override args\n ## @param git.sync.extraVolumeMounts Add extra volume mounts\n ## @param git.sync.extraEnvVars Add extra environment variables\n ## @param git.sync.extraEnvVarsCM ConfigMap with extra environment variables\n ## @param git.sync.extraEnvVarsSecret Secret with extra environment variables\n ## @param git.sync.resources Sync sidecar container resource requests and limits\n ##\n sync:\n interval: 60\n command: []\n args: []\n extraVolumeMounts: []\n extraEnvVars: []\n extraEnvVarsCM: \"\"\n extraEnvVarsSecret: \"\"\n ## Sync sidecar container resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ##\n resources: {}\n\n## @section Airflow ldap parameters\n\n## LDAP configuration\n## @param ldap.enabled Enable LDAP authentication\n## @param ldap.uri Server URI, eg. ldap://ldap_server:389\n## DEPRECATED ldap.base It will be removed in a future release, please use 'ldap.basedn' instead\n## @param ldap.basedn Base of the search, eg. ou=example,o=org.\n## DEPRECATED ldap.uidField It will be removed in a future release,, please use 'ldap.searchAttribute' instead\n## @param ldap.searchAttribute if doing an indirect bind to ldap, this is the field that matches the username when searching for the account to bind to\n## @param ldap.binddn DN of the account used to search in the LDAP server.\n## @param ldap.bindpw Bind Password\n## @param ldap.userRegistration Set to True to enable user self registration\n## @param ldap.userRegistrationRole Set role name to be assign when a user registers himself. This role must already exist. Mandatory when using ldap.userRegistration\n## @param ldap.rolesMapping mapping from LDAP DN to a list of roles\n## @param ldap.rolesSyncAtLogin replace ALL the user's roles each login, or only on registration\n##\nldap:\n enabled: false\n uri: \"ldap://ldap_server:389\"\n basedn: \"dc=example,dc=org\"\n searchAttribute: \"cn\"\n binddn: \"cn=admin,dc=example,dc=org\"\n bindpw: \"\"\n userRegistration: 'True'\n userRegistrationRole: \"Public\"\n rolesMapping: '{ \"cn=All,ou=Groups,dc=example,dc=org\": [\"User\"], \"cn=Admins,ou=Groups,dc=example,dc=org\": [\"Admin\"], }'\n rolesSyncAtLogin: 'True'\n\n ## SSL/TLS parameters for LDAP\n ## @param ldap.tls.enabled Enabled TLS/SSL for LDAP, you must include the CA file.\n ## @param ldap.tls.allowSelfSigned Allow to use self signed certificates\n ## DEPRECATED ldap.tls.CAcertificateSecret It will be removed in a future release, please use ldap.tls.certificatesSecret instead\n ## @param ldap.tls.certificatesSecret Name of the existing secret containing the certificate CA file that will be used by ldap client\n ## @param ldap.tls.certificatesMountPath Where LDAP certifcates are mounted.\n ## DEPRECATED ldap.tls.CAcertificateFilename It will be removed in a future release, please use ldap.tls.CAFilename instead\n ## @param ldap.tls.CAFilename LDAP CA cert filename\n ##\n tls:\n enabled: false\n allowSelfSigned: true\n certificatesSecret: \"\"\n certificatesMountPath: /opt/bitnami/airflow/conf/certs\n CAFilename: \"\"\n\n## @section Traffic Exposure Parameters\n\n## Airflow service parameters\n##\nservice:\n ## @param service.type Airflow service type\n ##\n type: ClusterIP\n ## @param service.ports.http Airflow service HTTP port\n ##\n ports:\n http: 8080\n ## Node ports to expose\n ## @param service.nodePorts.http Node port for HTTP\n ## NOTE: choose port between <30000-32767>\n ##\n nodePorts:\n http: \"\"\n ## @param service.sessionAffinity Control where client requests go, to the same pod or round-robin\n ## Values: ClientIP or None\n ## ref: https://kubernetes.io/docs/user-guide/services/\n ##\n sessionAffinity: None\n ## @param service.sessionAffinityConfig Additional settings for the sessionAffinity\n ## sessionAffinityConfig:\n ## clientIP:\n ## timeoutSeconds: 300\n ##\n sessionAffinityConfig: {}\n ## @param service.clusterIP Airflow service Cluster IP\n ## e.g.:\n ## clusterIP: None\n ##\n clusterIP: \"\"\n ## @param service.loadBalancerIP Airflow service Load Balancer IP\n ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-loadbalancer\n ##\n loadBalancerIP: \"\"\n ## @param service.loadBalancerSourceRanges Airflow service Load Balancer sources\n ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service\n ## e.g:\n ## loadBalancerSourceRanges:\n ## - 10.10.10.0/24\n ##\n loadBalancerSourceRanges: []\n ## @param service.externalTrafficPolicy Airflow service external traffic policy\n ## ref https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip\n ##\n externalTrafficPolicy: Cluster\n ## @param service.annotations Additional custom annotations for Airflow service\n ##\n annotations: {}\n ## @param service.extraPorts Extra port to expose on Airflow service\n ##\n extraPorts: []\n\n## Airflow ingress parameters\n## ref: https://kubernetes.io/docs/user-guide/ingress/\n##\ningress:\n ## @param ingress.enabled Enable ingress record generation for Airflow\n ##\n enabled: false\n ## @param ingress.ingressClassName IngressClass that will be be used to implement the Ingress (Kubernetes 1.18+)\n ## This is supported in Kubernetes 1.18+ and required if you have more than one IngressClass marked as the default for your cluster .\n ## ref: https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/\n ##\n ingressClassName: \"\"\n ## @param ingress.pathType Ingress path type\n ##\n pathType: ImplementationSpecific\n ## @param ingress.apiVersion Force Ingress API version (automatically detected if not set)\n ##\n apiVersion: \"\"\n ## @param ingress.hostname Default host for the ingress record\n ##\n hostname: airflow.local\n ## @param ingress.path Default path for the ingress record\n ## NOTE: You may need to set this to '/*' in order to use this with ALB ingress controllers\n ##\n path: /\n ## @param ingress.annotations [object] Additional annotations for the Ingress resource. To enable certificate autogeneration, place here your cert-manager annotations.\n ## Use this parameter to set the required annotations for cert-manager, see\n ## ref: https://cert-manager.io/docs/usage/ingress/#supported-annotations\n ## e.g:\n ## annotations:\n ## kubernetes.io/ingress.class: nginx\n ## cert-manager.io/cluster-issuer: cluster-issuer-name\n ##\n annotations: {}\n ## @param ingress.tls Enable TLS configuration for the host defined at `ingress.hostname` parameter\n ## TLS certificates will be retrieved from a TLS secret with name: `{{- printf \"%s-tls\" .Values.ingress.hostname }}`\n ## You can:\n ## - Use the `ingress.secrets` parameter to create this TLS secret\n ## - Rely on cert-manager to create it by setting the corresponding annotations\n ## - Rely on Helm to create self-signed certificates by setting `ingress.selfSigned=true`\n ##\n tls: false\n ## @param ingress.selfSigned Create a TLS secret for this ingress record using self-signed certificates generated by Helm\n ##\n selfSigned: false\n ## @param ingress.extraHosts An array with additional hostname(s) to be covered with the ingress record\n ## e.g:\n ## extraHosts:\n ## - name: airflow.local\n ## path: /\n ##\n extraHosts: []\n ## @param ingress.extraPaths An array with additional arbitrary paths that may need to be added to the ingress under the main host\n ## e.g:\n ## extraPaths:\n ## - path: /*\n ## backend:\n ## serviceName: ssl-redirect\n ## servicePort: use-annotation\n ##\n extraPaths: []\n ## @param ingress.extraTls TLS configuration for additional hostname(s) to be covered with this ingress record\n ## ref: https://kubernetes.io/docs/concepts/services-networking/ingress/#tls\n ## e.g:\n ## extraTls:\n ## - hosts:\n ## - airflow.local\n ## secretName: airflow.local-tls\n ##\n extraTls: []\n ## @param ingress.secrets Custom TLS certificates as secrets\n ## NOTE: 'key' and 'certificate' are expected in PEM format\n ## NOTE: 'name' should line up with a 'secretName' set further up\n ## If it is not set and you're using cert-manager, this is unneeded, as it will create a secret for you with valid certificates\n ## If it is not set and you're NOT using cert-manager either, self-signed certificates will be created valid for 365 days\n ## It is also possible to create and manage the certificates outside of this helm chart\n ## Please see README.md for more information\n ## e.g:\n ## secrets:\n ## - name: airflow.local-tls\n ## key: |-\n ## -----BEGIN RSA PRIVATE KEY-----\n ## ...\n ## -----END RSA PRIVATE KEY-----\n ## certificate: |-\n ## -----BEGIN CERTIFICATE-----\n ## ...\n ## -----END CERTIFICATE-----\n ##\n secrets: []\n ## @param ingress.extraRules Additional rules to be covered with this ingress record\n ## ref: https://kubernetes.io/docs/concepts/services-networking/ingress/#ingress-rules\n ## e.g:\n ## extraRules:\n ## - host: example.local\n ## http:\n ## path: /\n ## backend:\n ## service:\n ## name: example-svc\n ## port:\n ## name: http\n ##\n extraRules: []\n\n## @section Other Parameters\n\n## Service account for Airflow pods to use.\n## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/\n##\nserviceAccount:\n ## @param serviceAccount.create Enable creation of ServiceAccount for Airflow pods\n ##\n create: false\n ## @param serviceAccount.name The name of the ServiceAccount to use.\n ## If not set and create is true, a name is generated using the common.names.fullname template\n ##\n name: \"\"\n ## @param serviceAccount.automountServiceAccountToken Allows auto mount of ServiceAccountToken on the serviceAccount created\n ## Can be set to false if pods using this serviceAccount do not need to use K8s API\n ##\n automountServiceAccountToken: true\n ## @param serviceAccount.annotations Additional custom annotations for the ServiceAccount\n ##\n annotations: {}\n## Role Based Access\n## Ref: https://kubernetes.io/docs/admin/authorization/rbac/\n## @param rbac.create Create Role and RoleBinding\n##\nrbac:\n create: false\n ## @param rbac.rules Custom RBAC rules to set\n ## e.g:\n ## rules:\n ## - apiGroups:\n ## - \"\"\n ## resources:\n ## - pods\n ## verbs:\n ## - get\n ## - list\n ##\n rules: []\n\n## @section Airflow metrics parameters\n\nmetrics:\n ## @param metrics.enabled Whether or not to create a standalone Airflow exporter to expose Airflow metrics\n ##\n enabled: false\n ## Bitnami Airflow exporter image\n ## ref: https://hub.docker.com/r/bitnami/airflow-exporter/tags/\n ## @param metrics.image.registry Airflow exporter image registry\n ## @param metrics.image.repository Airflow exporter image repository\n ## @param metrics.image.tag Airflow exporter image tag (immutable tags are recommended)\n ## @param metrics.image.digest Airflow exporter image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param metrics.image.pullPolicy Airflow exporter image pull policy\n ## @param metrics.image.pullSecrets Airflow exporter image pull secrets\n ##\n image:\n registry: docker.io\n repository: bitnami/airflow-exporter\n tag: 0.20220314.0-debian-11-r57\n digest: \"\"\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## @param metrics.extraEnvVars Array with extra environment variables to add Airflow exporter pods\n ##\n extraEnvVars: []\n ## @param metrics.extraEnvVarsCM ConfigMap containing extra environment variables for Airflow exporter pods\n ##\n extraEnvVarsCM: \"\"\n ## @param metrics.extraEnvVarsSecret Secret containing extra environment variables (in case of sensitive data) for Airflow exporter pods\n ##\n extraEnvVarsSecret: \"\"\n ## @param metrics.containerPorts.http Airflow exporter metrics container port\n ##\n containerPorts:\n http: 9112\n ## Airflow exporter resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ## @param metrics.resources.limits The resources limits for the container\n ## @param metrics.resources.requests The requested resources for the container\n ##\n resources:\n limits: {}\n requests: {}\n ## Airflow exporter pods' Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod\n ## @param metrics.podSecurityContext.enabled Enable security context for the pods\n ## @param metrics.podSecurityContext.fsGroup Set Airflow exporter pod's Security Context fsGroup\n ##\n podSecurityContext:\n enabled: true\n fsGroup: 1001\n ## Airflow exporter containers' Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container\n ## @param metrics.containerSecurityContext.enabled Enable Airflow exporter containers' Security Context\n ## @param metrics.containerSecurityContext.runAsUser Set Airflow exporter containers' Security Context runAsUser\n ## @param metrics.containerSecurityContext.runAsNonRoot Set Airflow exporter containers' Security Context runAsNonRoot\n ## e.g:\n ## containerSecurityContext:\n ## enabled: true\n ## capabilities:\n ## drop: [\"NET_RAW\"]\n ## readOnlyRootFilesystem: true\n ##\n containerSecurityContext:\n enabled: true\n runAsUser: 1001\n runAsNonRoot: true\n ## @param metrics.lifecycleHooks for the Airflow exporter container(s) to automate configuration before or after startup\n ##\n lifecycleHooks: {}\n ## @param metrics.hostAliases Airflow exporter pods host aliases\n ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/\n ##\n hostAliases: []\n ## @param metrics.podLabels Extra labels for Airflow exporter pods\n ## Ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/\n ##\n podLabels: {}\n ## @param metrics.podAnnotations Extra annotations for Airflow exporter pods\n ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/\n ##\n podAnnotations: {}\n ## @param metrics.podAffinityPreset Pod affinity preset. Ignored if `metrics.affinity` is set. Allowed values: `soft` or `hard`\n ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAffinityPreset: \"\"\n ## @param metrics.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `metrics.affinity` is set. Allowed values: `soft` or `hard`\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAntiAffinityPreset: soft\n ## Node metrics.affinity preset\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity\n ##\n nodeAffinityPreset:\n ## @param metrics.nodeAffinityPreset.type Node affinity preset type. Ignored if `metrics.affinity` is set. Allowed values: `soft` or `hard`\n ##\n type: \"\"\n ## @param metrics.nodeAffinityPreset.key Node label key to match Ignored if `metrics.affinity` is set.\n ## E.g.\n ## key: \"kubernetes.io/e2e-az-name\"\n ##\n key: \"\"\n ## @param metrics.nodeAffinityPreset.values Node label values to match. Ignored if `metrics.affinity` is set.\n ## E.g.\n ## values:\n ## - e2e-az1\n ## - e2e-az2\n ##\n values: []\n ## @param metrics.affinity Affinity for pod assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity\n ## Note: metrics.podAffinityPreset, metrics.podAntiAffinityPreset, and metrics.nodeAffinityPreset will be ignored when it's set\n ##\n affinity: {}\n ## @param metrics.nodeSelector Node labels for pod assignment\n ## Ref: https://kubernetes.io/docs/user-guide/node-selection/\n ##\n nodeSelector: {}\n ## @param metrics.tolerations Tolerations for pod assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n ##\n tolerations: []\n ## @param metrics.schedulerName Name of the k8s scheduler (other than default) for Airflow exporter\n ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/\n ##\n schedulerName: \"\"\n ## Airflow exporter service configuration\n ##\n service:\n ## @param metrics.service.ports.http Airflow exporter metrics service port\n ##\n ports:\n http: 9112\n ## @param metrics.service.clusterIP Static clusterIP or None for headless services\n ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#choosing-your-own-ip-address\n ##\n clusterIP: \"\"\n ## @param metrics.service.sessionAffinity Control where client requests go, to the same pod or round-robin\n ## Values: ClientIP or None\n ## ref: https://kubernetes.io/docs/user-guide/services/\n ##\n sessionAffinity: None\n ## @param metrics.service.annotations [object] Annotations for the Airflow exporter service\n ##\n annotations:\n prometheus.io/scrape: \"true\"\n prometheus.io/port: \"{{ .Values.metrics.service.ports.http }}\"\n ## Prometheus Operator ServiceMonitor configuration\n ##\n serviceMonitor:\n ## @param metrics.serviceMonitor.enabled if `true`, creates a Prometheus Operator ServiceMonitor (requires `metrics.enabled` to be `true`)\n ##\n enabled: false\n ## @param metrics.serviceMonitor.namespace Namespace in which Prometheus is running\n ##\n namespace: \"\"\n ## @param metrics.serviceMonitor.interval Interval at which metrics should be scraped\n ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint\n ##\n interval: \"\"\n ## @param metrics.serviceMonitor.scrapeTimeout Timeout after which the scrape is ended\n ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint\n ##\n scrapeTimeout: \"\"\n ## @param metrics.serviceMonitor.labels Additional labels that can be used so ServiceMonitor will be discovered by Prometheus\n ##\n labels: {}\n ## @param metrics.serviceMonitor.selector Prometheus instance selector labels\n ## ref: https://github.com/bitnami/charts/tree/main/bitnami/prometheus-operator#prometheus-configuration\n ##\n selector: {}\n ## @param metrics.serviceMonitor.relabelings RelabelConfigs to apply to samples before scraping\n ##\n relabelings: []\n ## @param metrics.serviceMonitor.metricRelabelings MetricRelabelConfigs to apply to samples before ingestion\n ##\n metricRelabelings: []\n ## @param metrics.serviceMonitor.honorLabels Specify honorLabels parameter to add the scrape endpoint\n ##\n honorLabels: false\n ## @param metrics.serviceMonitor.jobLabel The name of the label on the target service to use as the job name in prometheus.\n ##\n jobLabel: \"\"\n\n## @section Airflow database parameters\n\n## PostgreSQL chart configuration\n## ref: https://github.com/bitnami/charts/blob/main/bitnami/postgresql/values.yaml\n## @param postgresql.enabled Switch to enable or disable the PostgreSQL helm chart\n## @param postgresql.auth.enablePostgresUser Assign a password to the \"postgres\" admin user. Otherwise, remote access will be blocked for this user\n## @param postgresql.auth.username Name for a custom user to create\n## @param postgresql.auth.password Password for the custom user to create\n## @param postgresql.auth.database Name for a custom database to create\n## @param postgresql.auth.existingSecret Name of existing secret to use for PostgreSQL credentials\n## @param postgresql.architecture PostgreSQL architecture (`standalone` or `replication`)\n##\npostgresql:\n enabled: true\n auth:\n enablePostgresUser: false\n username: bn_airflow\n password: \"\"\n database: bitnami_airflow\n existingSecret: \"\"\n architecture: standalone\n## External PostgreSQL configuration\n## All of these values are only used when postgresql.enabled is set to false\n## @param externalDatabase.host Database host\n## @param externalDatabase.port Database port number\n## @param externalDatabase.user Non-root username for Airflow\n## @param externalDatabase.password Password for the non-root username for Airflow\n## @param externalDatabase.database Airflow database name\n## @param externalDatabase.existingSecret Name of an existing secret resource containing the database credentials\n## @param externalDatabase.existingSecretPasswordKey Name of an existing secret key containing the database credentials\n##\nexternalDatabase:\n host: localhost\n port: 5432\n user: bn_airflow\n database: bitnami_airflow\n password: \"\"\n existingSecret: \"\"\n existingSecretPasswordKey: \"\"\n\n## Redis® chart configuration\n## ref: https://github.com/bitnami/charts/blob/main/bitnami/redis/values.yaml\n## @param redis.enabled Switch to enable or disable the Redis® helm\n## @param redis.auth.enabled Enable password authentication\n## @param redis.auth.password Redis® password\n## @param redis.auth.existingSecret The name of an existing secret with Redis® credentials\n## @param redis.architecture Redis® architecture. Allowed values: `standalone` or `replication`\n##\nredis:\n enabled: true\n auth:\n enabled: true\n ## Redis® password (both master and slave). Defaults to a random 10-character alphanumeric string if not set and auth.enabled is true.\n ## It should always be set using the password value or in the existingSecret to avoid issues\n ## with Airflow.\n ## The password value is ignored if existingSecret is set\n password: \"\"\n existingSecret: \"\"\n architecture: standalone\n\n## External Redis® configuration\n## All of these values are only used when redis.enabled is set to false\n## @param externalRedis.host Redis® host\n## @param externalRedis.port Redis® port number\n## @param externalRedis.username Redis® username\n## @param externalRedis.password Redis® password\n## @param externalRedis.existingSecret Name of an existing secret resource containing the Redis&trade credentials\n## @param externalRedis.existingSecretPasswordKey Name of an existing secret key containing the Redis&trade credentials\n##\nexternalRedis:\n host: localhost\n port: 6379\n ## Most Redis® implementations do not require a username\n ## to authenticate and it should be enough with the password\n username: \"\"\n password: \"\"\n existingSecret: \"\"\n existingSecretPasswordKey: \"\"\n", Readme: "", UserId: 6, ReferenceValueId: 6304, @@ -93,7 +93,7 @@ func TestAppStoreDeploymentService(t *testing.T) { InstalledAppId: 0, InstalledAppVersionId: 0, AppStoreVersion: 6304, - ValuesOverrideYaml: "## @section Global parameters\n## Global Docker image parameters\n## Please, note that this will override the image parameters, including dependencies, configured to use the global value\n## Current available global Docker image parameters: imageRegistry, imagePullSecrets and storageClass\n\n## @param global.imageRegistry Global Docker image registry\n## @param global.imagePullSecrets Global Docker registry secret names as an array\n## @param global.storageClass Global StorageClass for Persistent Volume(s)\n##\nglobal:\n imageRegistry: \"\"\n ## E.g.\n ## imagePullSecrets:\n ## - myRegistryKeySecretName\n ##\n imagePullSecrets: []\n storageClass: \"\"\n\n## @section Common parameters\n\n## @param kubeVersion Override Kubernetes version\n##\nkubeVersion: \"\"\n## @param nameOverride String to partially override common.names.fullname template (will maintain the release name)\n##\nnameOverride: \"\"\n## @param fullnameOverride String to fully override common.names.fullname template\n##\nfullnameOverride: \"\"\n## @param clusterDomain Kubernetes Cluster Domain\n##\nclusterDomain: cluster.local\n## @param extraDeploy Extra objects to deploy (evaluated as a template)\n##\nextraDeploy: []\n## @param commonLabels Add labels to all the deployed resources\n##\ncommonLabels: {}\n## @param commonAnnotations Add annotations to all the deployed resources\n##\ncommonAnnotations: {}\n## Enable diagnostic mode in the deployment(s)/statefulset(s)\n##\ndiagnosticMode:\n ## @param diagnosticMode.enabled Enable diagnostic mode (all probes will be disabled and the command will be overridden)\n ##\n enabled: false\n ## @param diagnosticMode.command Command to override all containers in the the deployment(s)/statefulset(s)\n ##\n command:\n - sleep\n ## @param diagnosticMode.args Args to override all containers in the the deployment(s)/statefulset(s)\n ##\n args:\n - infinity\n\n## @section Airflow common parameters\n\n## Authentication parameters\n## ref: https://github.com/bitnami/containers/tree/main/bitnami/airflow#environment-variables\n##\nauth:\n ## @param auth.username Username to access web UI\n ##\n username: user\n ## @param auth.password Password to access web UI\n ##\n password: \"\"\n ## @param auth.fernetKey Fernet key to secure connections\n ## ref: https://airflow.readthedocs.io/en/stable/howto/secure-connections.html\n ## ref: https://bcb.github.io/airflow/fernet-key\n ##\n fernetKey: \"\"\n ## @param auth.secretKey Secret key to run your flask app\n ## ref: https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.html#secret-key\n ##\n secretKey: \"\"\n ## @param auth.existingSecret Name of an existing secret to use for Airflow credentials\n ## `auth.password`, `auth.fernetKey`, and `auth.secretKey` will be ignored and picked up from this secret\n ## The secret must contain the keys `airflow-password`, `airflow-fernet-key` and `airflow-secret-key'\n ## The value is evaluated as a template\n ##\n existingSecret: \"\"\n## @param executor Airflow executor. Allowed values: `SequentialExecutor`, `LocalExecutor`, `CeleryExecutor`, `KubernetesExecutor`, `CeleryKubernetesExecutor` and `LocalKubernetesExecutor`\n## ref: http://airflow.apache.org/docs/stable/executor/index.html\n##\nexecutor: CeleryExecutor\n## @param loadExamples Switch to load some Airflow examples\n##\nloadExamples: false\n## @param configuration Specify content for Airflow config file (auto-generated based on other env. vars otherwise)\n## e.g:\n## configuration: |-\n## [core]\n## dags_folder=/opt/bitnami/airflow/dags\n## ...\n##\nconfiguration: \"\"\n## @param existingConfigmap Name of an existing ConfigMap with the Airflow config file\n##\nexistingConfigmap: \"\"\n## Load custom DAGs from a ConfigMap\n## Note: an init container will be used to prepare the DAGs available in the ConfigMap to be consumed by Airflow containers\n##\ndags:\n ## @param dags.existingConfigmap Name of an existing ConfigMap with all the DAGs files you want to load in Airflow\n ##\n existingConfigmap: \"\"\n ## Bitnami Shell image\n ## ref: https://hub.docker.com/r/bitnami/bitnami-shell/tags/\n ## @param dags.image.registry Init container load-dags image registry\n ## @param dags.image.repository Init container load-dags image repository\n ## @param dags.image.tag Init container load-dags image tag (immutable tags are recommended)\n ## @param dags.image.digest Init container load-dags image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param dags.image.pullPolicy Init container load-dags image pull policy\n ## @param dags.image.pullSecrets Init container load-dags image pull secrets\n ##\n image:\n registry: docker.io\n repository: bitnami/bitnami-shell\n tag: 11-debian-11-r50\n digest: \"\"\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n## @param extraEnvVars Add extra environment variables for all the Airflow pods\n##\nextraEnvVars: []\n## @param extraEnvVarsCM ConfigMap with extra environment variables for all the Airflow pods\n##\nextraEnvVarsCM: \"\"\n## @param extraEnvVarsSecret Secret with extra environment variables for all the Airflow pods\n##\nextraEnvVarsSecret: \"\"\n## @param extraEnvVarsSecrets List of secrets with extra environment variables for all the Airflow pods\n##\nextraEnvVarsSecrets: []\n## @param sidecars Add additional sidecar containers to all the Airflow pods\n## Example:\n## sidecars:\n## - name: your-image-name\n## image: your-image\n## imagePullPolicy: Always\n## ports:\n## - name: portname\n## containerPort: 1234\n##\nsidecars: []\n## @param initContainers Add additional init containers to all the Airflow pods\n## Example:\n## initContainers:\n## - name: your-image-name\n## image: your-image\n## imagePullPolicy: Always\n## ports:\n## - name: portname\n## containerPort: 1234\n##\ninitContainers: []\n## @param extraVolumeMounts Optionally specify extra list of additional volumeMounts for all the Airflow pods\n##\nextraVolumeMounts: []\n## @param extraVolumes Optionally specify extra list of additional volumes for the all the Airflow pods\n##\nextraVolumes: []\n\n## @section Airflow web parameters\n\nweb:\n ## Bitnami Airflow image version\n ## ref: https://hub.docker.com/r/bitnami/airflow/tags/\n ## @param web.image.registry Airflow image registry\n ## @param web.image.repository Airflow image repository\n ## @param web.image.tag Airflow image tag (immutable tags are recommended)\n ## @param web.image.digest Airflow image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param web.image.pullPolicy Airflow image pull policy\n ## @param web.image.pullSecrets Airflow image pull secrets\n ## @param web.image.debug Enable image debug mode\n image:\n registry: docker.io\n repository: bitnami/airflow\n tag: 2.4.2-debian-11-r6\n digest: \"\"\n ## Specify a imagePullPolicy\n ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'\n ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images\n ##\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## Set to true if you would like to see extra information on logs\n ##\n debug: false\n ## @param web.baseUrl URL used to access to Airflow web ui\n ##\n baseUrl: \"\"\n ## @param web.existingConfigmap Name of an existing config map containing the Airflow web config file\n ##\n existingConfigmap: \"\"\n ## @param web.command Override default container command (useful when using custom images)\n ##\n command: []\n ## @param web.args Override default container args (useful when using custom images)\n ##\n args: []\n ## @param web.extraEnvVars Array with extra environment variables to add Airflow web pods\n ##\n extraEnvVars: []\n ## @param web.extraEnvVarsCM ConfigMap containing extra environment variables for Airflow web pods\n ##\n extraEnvVarsCM: \"\"\n ## @param web.extraEnvVarsSecret Secret containing extra environment variables (in case of sensitive data) for Airflow web pods\n ##\n extraEnvVarsSecret: \"\"\n ## @param web.extraEnvVarsSecrets List of secrets with extra environment variables for Airflow web pods\n ##\n extraEnvVarsSecrets: []\n ## @param web.containerPorts.http Airflow web HTTP container port\n ##\n containerPorts:\n http: 8080\n ## @param web.replicaCount Number of Airflow web replicas\n ##\n replicaCount: 1\n ## Configure extra options for Airflow web containers' liveness, readiness and startup probes\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes\n ## @param web.livenessProbe.enabled Enable livenessProbe on Airflow web containers\n ## @param web.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe\n ## @param web.livenessProbe.periodSeconds Period seconds for livenessProbe\n ## @param web.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe\n ## @param web.livenessProbe.failureThreshold Failure threshold for livenessProbe\n ## @param web.livenessProbe.successThreshold Success threshold for livenessProbe\n ##\n livenessProbe:\n enabled: true\n initialDelaySeconds: 180\n periodSeconds: 20\n timeoutSeconds: 5\n failureThreshold: 6\n successThreshold: 1\n ## @param web.readinessProbe.enabled Enable readinessProbe on Airflow web containers\n ## @param web.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe\n ## @param web.readinessProbe.periodSeconds Period seconds for readinessProbe\n ## @param web.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe\n ## @param web.readinessProbe.failureThreshold Failure threshold for readinessProbe\n ## @param web.readinessProbe.successThreshold Success threshold for readinessProbe\n ##\n readinessProbe:\n enabled: true\n initialDelaySeconds: 30\n periodSeconds: 10\n timeoutSeconds: 5\n failureThreshold: 6\n successThreshold: 1\n ## @param web.startupProbe.enabled Enable startupProbe on Airflow web containers\n ## @param web.startupProbe.initialDelaySeconds Initial delay seconds for startupProbe\n ## @param web.startupProbe.periodSeconds Period seconds for startupProbe\n ## @param web.startupProbe.timeoutSeconds Timeout seconds for startupProbe\n ## @param web.startupProbe.failureThreshold Failure threshold for startupProbe\n ## @param web.startupProbe.successThreshold Success threshold for startupProbe\n ##\n startupProbe:\n enabled: false\n initialDelaySeconds: 60\n periodSeconds: 10\n timeoutSeconds: 1\n failureThreshold: 15\n successThreshold: 1\n ## @param web.customLivenessProbe Custom livenessProbe that overrides the default one\n ##\n customLivenessProbe: {}\n ## @param web.customReadinessProbe Custom readinessProbe that overrides the default one\n ##\n customReadinessProbe: {}\n ## @param web.customStartupProbe Custom startupProbe that overrides the default one\n ##\n customStartupProbe: {}\n ## Airflow web resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ## @param web.resources.limits The resources limits for the Airflow web containers\n ## @param web.resources.requests The requested resources for the Airflow web containers\n ##\n resources:\n limits: {}\n requests: {}\n ## Configure Airflow web pods Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod\n ## @param web.podSecurityContext.enabled Enabled Airflow web pods' Security Context\n ## @param web.podSecurityContext.fsGroup Set Airflow web pod's Security Context fsGroup\n ##\n podSecurityContext:\n enabled: true\n fsGroup: 1001\n ## Configure Airflow web containers (only main one) Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container\n ## @param web.containerSecurityContext.enabled Enabled Airflow web containers' Security Context\n ## @param web.containerSecurityContext.runAsUser Set Airflow web containers' Security Context runAsUser\n ## @param web.containerSecurityContext.runAsNonRoot Set Airflow web containers' Security Context runAsNonRoot\n ##\n containerSecurityContext:\n enabled: true\n runAsUser: 1001\n runAsNonRoot: true\n ## @param web.lifecycleHooks for the Airflow web container(s) to automate configuration before or after startup\n ##\n lifecycleHooks: {}\n ## @param web.hostAliases Deployment pod host aliases\n ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/\n ##\n hostAliases: []\n ## @param web.podLabels Add extra labels to the Airflow web pods\n ##\n podLabels: {}\n ## @param web.podAnnotations Add extra annotations to the Airflow web pods\n ##\n podAnnotations: {}\n ## @param web.affinity Affinity for Airflow web pods assignment (evaluated as a template)\n ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity\n ## Note: `web.podAffinityPreset`, `web.podAntiAffinityPreset`, and `web.nodeAffinityPreset` will be ignored when it's set\n ##\n affinity: {}\n ## Node affinity preset\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity\n ## @param web.nodeAffinityPreset.key Node label key to match. Ignored if `web.affinity` is set.\n ## @param web.nodeAffinityPreset.type Node affinity preset type. Ignored if `web.affinity` is set. Allowed values: `soft` or `hard`\n ## @param web.nodeAffinityPreset.values Node label values to match. Ignored if `web.affinity` is set.\n ##\n nodeAffinityPreset:\n ## e.g:\n ## key: \"kubernetes.io/e2e-az-name\"\n ##\n key: \"\"\n type: \"\"\n ## e.g:\n ## values:\n ## - e2e-az1\n ## - e2e-az2\n ##\n values: []\n ## @param web.nodeSelector Node labels for Airflow web pods assignment\n ## Ref: https://kubernetes.io/docs/user-guide/node-selection/\n ##\n nodeSelector: {}\n ## @param web.podAffinityPreset Pod affinity preset. Ignored if `web.affinity` is set. Allowed values: `soft` or `hard`.\n ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAffinityPreset: \"\"\n ## @param web.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `web.affinity` is set. Allowed values: `soft` or `hard`.\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAntiAffinityPreset: soft\n ## @param web.tolerations Tolerations for Airflow web pods assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n ##\n tolerations: []\n ## @param web.topologySpreadConstraints Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template\n ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods\n ##\n topologySpreadConstraints: []\n ## @param web.priorityClassName Priority Class Name\n ## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass\n ##\n priorityClassName: \"\"\n ## @param web.schedulerName Use an alternate scheduler, e.g. \"stork\".\n ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/\n ##\n schedulerName: \"\"\n ## @param web.terminationGracePeriodSeconds Seconds Airflow web pod needs to terminate gracefully\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods\n ##\n terminationGracePeriodSeconds: \"\"\n ## @param web.updateStrategy.type Airflow web deployment strategy type\n ## @param web.updateStrategy.rollingUpdate Airflow web deployment rolling update configuration parameters\n ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy\n ##\n updateStrategy:\n type: RollingUpdate\n rollingUpdate: {}\n ## @param web.sidecars Add additional sidecar containers to the Airflow web pods\n ## Example:\n ## sidecars:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n sidecars: []\n ## @param web.initContainers Add additional init containers to the Airflow web pods\n ## Example:\n ## initContainers:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n initContainers: []\n ## @param web.extraVolumeMounts Optionally specify extra list of additional volumeMounts for the Airflow web pods\n ##\n extraVolumeMounts: []\n ## @param web.extraVolumes Optionally specify extra list of additional volumes for the Airflow web pods\n ##\n extraVolumes: []\n ## Airflow web Pod Disruption Budget\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/\n ## @param web.pdb.create Deploy a pdb object for the Airflow web pods\n ## @param web.pdb.minAvailable Maximum number/percentage of unavailable Airflow web replicas\n ## @param web.pdb.maxUnavailable Maximum number/percentage of unavailable Airflow web replicas\n ##\n pdb:\n create: false\n minAvailable: 1\n maxUnavailable: \"\"\n\n## @section Airflow scheduler parameters\n\nscheduler:\n ## Bitnami Airflow Scheduler image version\n ## ref: https://hub.docker.com/r/bitnami/airflow-scheduler/tags/\n ## @param scheduler.image.registry Airflow Scheduler image registry\n ## @param scheduler.image.repository Airflow Scheduler image repository\n ## @param scheduler.image.tag Airflow Scheduler image tag (immutable tags are recommended)\n ## @param scheduler.image.digest Airflow Schefuler image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param scheduler.image.pullPolicy Airflow Scheduler image pull policy\n ## @param scheduler.image.pullSecrets Airflow Scheduler image pull secrets\n ## @param scheduler.image.debug Enable image debug mode\n ##\n image:\n registry: docker.io\n repository: bitnami/airflow-scheduler\n tag: 2.4.2-debian-11-r4\n digest: \"\"\n ## Specify a imagePullPolicy\n ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'\n ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images\n ##\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## Set to true if you would like to see extra information on logs\n ##\n debug: false\n ## @param scheduler.replicaCount Number of scheduler replicas\n ##\n replicaCount: 1\n ## @param scheduler.command Override cmd\n ##\n command: []\n ## @param scheduler.args Override args\n ##\n args: []\n ## @param scheduler.extraEnvVars Add extra environment variables\n ##\n extraEnvVars: []\n ## @param scheduler.extraEnvVarsCM ConfigMap with extra environment variables\n ##\n extraEnvVarsCM: \"\"\n ## @param scheduler.extraEnvVarsSecret Secret with extra environment variables\n ##\n extraEnvVarsSecret: \"\"\n ## @param scheduler.extraEnvVarsSecrets List of secrets with extra environment variables for Airflow scheduler pods\n ##\n extraEnvVarsSecrets: []\n ## @param scheduler.customLivenessProbe Custom livenessProbe that overrides the default one\n ##\n customLivenessProbe: {}\n ## @param scheduler.customReadinessProbe Custom readinessProbe that overrides the default one\n ##\n customReadinessProbe: {}\n ## @param scheduler.customStartupProbe Custom startupProbe that overrides the default one\n ##\n customStartupProbe: {}\n ## Airflow scheduler resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ## @param scheduler.resources.limits The resources limits for the Airflow scheduler containers\n ## @param scheduler.resources.requests The requested resources for the Airflow scheduler containers\n ##\n resources:\n limits: {}\n requests: {}\n ## Configure Airflow scheduler pods Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod\n ## @param scheduler.podSecurityContext.enabled Enabled Airflow scheduler pods' Security Context\n ## @param scheduler.podSecurityContext.fsGroup Set Airflow scheduler pod's Security Context fsGroup\n ##\n podSecurityContext:\n enabled: true\n fsGroup: 1001\n ## Configure Airflow scheduler containers (only main one) Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container\n ## @param scheduler.containerSecurityContext.enabled Enabled Airflow scheduler containers' Security Context\n ## @param scheduler.containerSecurityContext.runAsUser Set Airflow scheduler containers' Security Context runAsUser\n ## @param scheduler.containerSecurityContext.runAsNonRoot Set Airflow scheduler containers' Security Context runAsNonRoot\n ##\n containerSecurityContext:\n enabled: true\n runAsUser: 1001\n runAsNonRoot: true\n ## @param scheduler.lifecycleHooks for the Airflow scheduler container(s) to automate configuration before or after startup\n ##\n lifecycleHooks: {}\n ## @param scheduler.hostAliases Deployment pod host aliases\n ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/\n ##\n hostAliases: []\n ## @param scheduler.podLabels Add extra labels to the Airflow scheduler pods\n ##\n podLabels: {}\n ## @param scheduler.podAnnotations Add extra annotations to the Airflow scheduler pods\n ##\n podAnnotations: {}\n ## @param scheduler.affinity Affinity for Airflow scheduler pods assignment (evaluated as a template)\n ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity\n ## Note: `scheduler.podAffinityPreset`, `scheduler.podAntiAffinityPreset`, and `scheduler.nodeAffinityPreset` will be ignored when it's set\n ##\n affinity: {}\n ## Node affinity preset\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity\n ## @param scheduler.nodeAffinityPreset.key Node label key to match. Ignored if `scheduler.affinity` is set.\n ## @param scheduler.nodeAffinityPreset.type Node affinity preset type. Ignored if `scheduler.affinity` is set. Allowed values: `soft` or `hard`\n ## @param scheduler.nodeAffinityPreset.values Node label values to match. Ignored if `scheduler.affinity` is set.\n ##\n nodeAffinityPreset:\n ## e.g:\n ## key: \"kubernetes.io/e2e-az-name\"\n ##\n key: \"\"\n type: \"\"\n ## e.g:\n ## values:\n ## - e2e-az1\n ## - e2e-az2\n ##\n values: []\n ## @param scheduler.nodeSelector Node labels for Airflow scheduler pods assignment\n ## Ref: https://kubernetes.io/docs/user-guide/node-selection/\n ##\n nodeSelector: {}\n ## @param scheduler.podAffinityPreset Pod affinity preset. Ignored if `scheduler.affinity` is set. Allowed values: `soft` or `hard`.\n ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAffinityPreset: \"\"\n ## @param scheduler.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `scheduler.affinity` is set. Allowed values: `soft` or `hard`.\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAntiAffinityPreset: soft\n ## @param scheduler.tolerations Tolerations for Airflow scheduler pods assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n ##\n tolerations: []\n ## @param scheduler.topologySpreadConstraints Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template\n ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods\n ##\n topologySpreadConstraints: []\n ## @param scheduler.priorityClassName Priority Class Name\n ## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass\n ##\n priorityClassName: \"\"\n ## @param scheduler.schedulerName Use an alternate scheduler, e.g. \"stork\".\n ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/\n ##\n schedulerName: \"\"\n ## @param scheduler.terminationGracePeriodSeconds Seconds Airflow scheduler pod needs to terminate gracefully\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods\n ##\n terminationGracePeriodSeconds: \"\"\n ## @param scheduler.updateStrategy.type Airflow scheduler deployment strategy type\n ## @param scheduler.updateStrategy.rollingUpdate Airflow scheduler deployment rolling update configuration parameters\n ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy\n ##\n updateStrategy:\n type: RollingUpdate\n rollingUpdate: {}\n ## @param scheduler.sidecars Add additional sidecar containers to the Airflow scheduler pods\n ## Example:\n ## sidecars:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n sidecars: []\n ## @param scheduler.initContainers Add additional init containers to the Airflow scheduler pods\n ## Example:\n ## initContainers:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n initContainers: []\n ## @param scheduler.extraVolumeMounts Optionally specify extra list of additional volumeMounts for the Airflow scheduler pods\n ##\n extraVolumeMounts: []\n ## @param scheduler.extraVolumes Optionally specify extra list of additional volumes for the Airflow scheduler pods\n ##\n extraVolumes: []\n ## Airflow scheduler Pod Disruption Budget\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/\n ## @param scheduler.pdb.create Deploy a pdb object for the Airflow scheduler pods\n ## @param scheduler.pdb.minAvailable Maximum number/percentage of unavailable Airflow scheduler replicas\n ## @param scheduler.pdb.maxUnavailable Maximum number/percentage of unavailable Airflow scheduler replicas\n ##\n pdb:\n create: false\n minAvailable: 1\n maxUnavailable: \"\"\n\n## @section Airflow worker parameters\n\nworker:\n ## Bitnami Airflow Worker image version\n ## ref: https://hub.docker.com/r/bitnami/airflow-worker/tags/\n ## @param worker.image.registry Airflow Worker image registry\n ## @param worker.image.repository Airflow Worker image repository\n ## @param worker.image.tag Airflow Worker image tag (immutable tags are recommended)\n ## @param worker.image.digest Airflow Worker image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param worker.image.pullPolicy Airflow Worker image pull policy\n ## @param worker.image.pullSecrets Airflow Worker image pull secrets\n ## @param worker.image.debug Enable image debug mode\n ##\n image:\n registry: docker.io\n repository: bitnami/airflow-worker\n tag: 2.4.2-debian-11-r4\n digest: \"\"\n ## Specify a imagePullPolicy\n ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'\n ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images\n ##\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## Set to true if you would like to see extra information on logs\n ##\n debug: false\n ## @param worker.command Override default container command (useful when using custom images)\n ##\n command: []\n ## @param worker.args Override default container args (useful when using custom images)\n ##\n args: []\n ## @param worker.extraEnvVars Array with extra environment variables to add Airflow worker pods\n ##\n extraEnvVars: []\n ## @param worker.extraEnvVarsCM ConfigMap containing extra environment variables for Airflow worker pods\n ##\n extraEnvVarsCM: \"\"\n ## @param worker.extraEnvVarsSecret Secret containing extra environment variables (in case of sensitive data) for Airflow worker pods\n ##\n extraEnvVarsSecret: \"\"\n ## @param worker.extraEnvVarsSecrets List of secrets with extra environment variables for Airflow worker pods\n ##\n extraEnvVarsSecrets: []\n ## @param worker.containerPorts.http Airflow worker HTTP container port\n ##\n containerPorts:\n http: 8793\n ## @param worker.replicaCount Number of Airflow worker replicas\n ##\n replicaCount: 1\n ## Configure extra options for Airflow worker containers' liveness, readiness and startup probes\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes\n ## @param worker.livenessProbe.enabled Enable livenessProbe on Airflow worker containers\n ## @param worker.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe\n ## @param worker.livenessProbe.periodSeconds Period seconds for livenessProbe\n ## @param worker.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe\n ## @param worker.livenessProbe.failureThreshold Failure threshold for livenessProbe\n ## @param worker.livenessProbe.successThreshold Success threshold for livenessProbe\n ##\n livenessProbe:\n enabled: true\n initialDelaySeconds: 180\n periodSeconds: 20\n timeoutSeconds: 5\n failureThreshold: 6\n successThreshold: 1\n ## @param worker.readinessProbe.enabled Enable readinessProbe on Airflow worker containers\n ## @param worker.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe\n ## @param worker.readinessProbe.periodSeconds Period seconds for readinessProbe\n ## @param worker.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe\n ## @param worker.readinessProbe.failureThreshold Failure threshold for readinessProbe\n ## @param worker.readinessProbe.successThreshold Success threshold for readinessProbe\n ##\n readinessProbe:\n enabled: true\n initialDelaySeconds: 30\n periodSeconds: 10\n timeoutSeconds: 5\n failureThreshold: 6\n successThreshold: 1\n ## @param worker.startupProbe.enabled Enable startupProbe on Airflow worker containers\n ## @param worker.startupProbe.initialDelaySeconds Initial delay seconds for startupProbe\n ## @param worker.startupProbe.periodSeconds Period seconds for startupProbe\n ## @param worker.startupProbe.timeoutSeconds Timeout seconds for startupProbe\n ## @param worker.startupProbe.failureThreshold Failure threshold for startupProbe\n ## @param worker.startupProbe.successThreshold Success threshold for startupProbe\n ##\n startupProbe:\n enabled: false\n initialDelaySeconds: 60\n periodSeconds: 10\n timeoutSeconds: 1\n failureThreshold: 15\n successThreshold: 1\n ## @param worker.customLivenessProbe Custom livenessProbe that overrides the default one\n ##\n customLivenessProbe: {}\n ## @param worker.customReadinessProbe Custom readinessProbe that overrides the default one\n ##\n customReadinessProbe: {}\n ## @param worker.customStartupProbe Custom startupProbe that overrides the default one\n ##\n customStartupProbe: {}\n ## Airflow worker resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ## @param worker.resources.limits The resources limits for the Airflow worker containers\n ## @param worker.resources.requests The requested resources for the Airflow worker containers\n ##\n resources:\n limits: {}\n requests: {}\n ## Configure Airflow worker pods Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod\n ## @param worker.podSecurityContext.enabled Enabled Airflow worker pods' Security Context\n ## @param worker.podSecurityContext.fsGroup Set Airflow worker pod's Security Context fsGroup\n ##\n podSecurityContext:\n enabled: true\n fsGroup: 1001\n ## Configure Airflow worker containers (only main one) Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container\n ## @param worker.containerSecurityContext.enabled Enabled Airflow worker containers' Security Context\n ## @param worker.containerSecurityContext.runAsUser Set Airflow worker containers' Security Context runAsUser\n ## @param worker.containerSecurityContext.runAsNonRoot Set Airflow worker containers' Security Context runAsNonRoot\n ##\n containerSecurityContext:\n enabled: true\n runAsUser: 1001\n runAsNonRoot: true\n ## @param worker.lifecycleHooks for the Airflow worker container(s) to automate configuration before or after startup\n ##\n lifecycleHooks: {}\n ## @param worker.hostAliases Deployment pod host aliases\n ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/\n ##\n hostAliases: []\n ## @param worker.podLabels Add extra labels to the Airflow worker pods\n ##\n podLabels: {}\n ## @param worker.podAnnotations Add extra annotations to the Airflow worker pods\n ##\n podAnnotations: {}\n ## @param worker.affinity Affinity for Airflow worker pods assignment (evaluated as a template)\n ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity\n ## Note: `worker.podAffinityPreset`, `worker.podAntiAffinityPreset`, and `worker.nodeAffinityPreset` will be ignored when it's set\n ##\n affinity: {}\n ## Node affinity preset\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity\n ## @param worker.nodeAffinityPreset.key Node label key to match. Ignored if `worker.affinity` is set.\n ## @param worker.nodeAffinityPreset.type Node affinity preset type. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard`\n ## @param worker.nodeAffinityPreset.values Node label values to match. Ignored if `worker.affinity` is set.\n ##\n nodeAffinityPreset:\n ## e.g:\n ## key: \"kubernetes.io/e2e-az-name\"\n ##\n key: \"\"\n type: \"\"\n ## e.g:\n ## values:\n ## - e2e-az1\n ## - e2e-az2\n ##\n values: []\n ## @param worker.nodeSelector Node labels for Airflow worker pods assignment\n ## Ref: https://kubernetes.io/docs/user-guide/node-selection/\n ##\n nodeSelector: {}\n ## @param worker.podAffinityPreset Pod affinity preset. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard`.\n ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAffinityPreset: \"\"\n ## @param worker.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard`.\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAntiAffinityPreset: soft\n ## @param worker.tolerations Tolerations for Airflow worker pods assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n ##\n tolerations: []\n ## @param worker.topologySpreadConstraints Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template\n ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods\n ##\n topologySpreadConstraints: []\n ## @param worker.priorityClassName Priority Class Name\n ## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass\n ##\n priorityClassName: \"\"\n ## @param worker.schedulerName Use an alternate scheduler, e.g. \"stork\".\n ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/\n ##\n schedulerName: \"\"\n ## @param worker.terminationGracePeriodSeconds Seconds Airflow worker pod needs to terminate gracefully\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods\n ##\n terminationGracePeriodSeconds: \"\"\n ## @param worker.updateStrategy.type Airflow worker deployment strategy type\n ## @param worker.updateStrategy.rollingUpdate Airflow worker deployment rolling update configuration parameters\n ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy\n ##\n updateStrategy:\n type: RollingUpdate\n rollingUpdate: {}\n ## @param worker.sidecars Add additional sidecar containers to the Airflow worker pods\n ## Example:\n ## sidecars:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n sidecars: []\n ## @param worker.initContainers Add additional init containers to the Airflow worker pods\n ## Example:\n ## initContainers:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n initContainers: []\n ## @param worker.extraVolumeMounts Optionally specify extra list of additional volumeMounts for the Airflow worker pods\n ##\n extraVolumeMounts: []\n ## @param worker.extraVolumes Optionally specify extra list of additional volumes for the Airflow worker pods\n ##\n extraVolumes: []\n ## @param worker.extraVolumeClaimTemplates Optionally specify extra list of volumesClaimTemplates for the Airflow worker statefulset\n ##\n extraVolumeClaimTemplates: []\n ## @param worker.podTemplate Template to replace the default one to be use when `executor=KubernetesExecutor` to create Airflow worker pods\n ##\n podTemplate: {}\n ## Airflow worker Pod Disruption Budget\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/\n ## @param worker.pdb.create Deploy a pdb object for the Airflow worker pods\n ## @param worker.pdb.minAvailable Maximum number/percentage of unavailable Airflow worker replicas\n ## @param worker.pdb.maxUnavailable Maximum number/percentage of unavailable Airflow worker replicas\n ##\n pdb:\n create: false\n minAvailable: 1\n maxUnavailable: \"\"\n ## Enable HorizontalPodAutoscaler for worker pods\n ## ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/\n ## @param worker.autoscaling.enabled Whether enable horizontal pod autoscaler\n ## @param worker.autoscaling.minReplicas Configure a minimum amount of pods\n ## @param worker.autoscaling.maxReplicas Configure a maximum amount of pods\n ## @param worker.autoscaling.targetCPU Define the CPU target to trigger the scaling actions (utilization percentage)\n ## @param worker.autoscaling.targetMemory Define the memory target to trigger the scaling actions (utilization percentage)\n ##\n autoscaling:\n enabled: false\n minReplicas: 1\n maxReplicas: 3\n targetCPU: 80\n targetMemory: 80\n\n## @section Airflow git sync parameters\n\n## Configure Git to pull dags and plugins\n##\ngit:\n ## Bitnami Git image version\n ## ref: https://hub.docker.com/r/bitnami/git/tags/\n ## @param git.image.registry Git image registry\n ## @param git.image.repository Git image repository\n ## @param git.image.tag Git image tag (immutable tags are recommended)\n ## @param git.image.digest Git image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param git.image.pullPolicy Git image pull policy\n ## @param git.image.pullSecrets Git image pull secrets\n ##\n image:\n registry: docker.io\n repository: bitnami/git\n tag: 2.38.1-debian-11-r7\n digest: \"\"\n ## Specify a imagePullPolicy\n ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'\n ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images\n ##\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## Get DAG files from git repositories\n ## @param git.dags.enabled Enable in order to download DAG files from git repositories.\n ## @param git.dags.repositories [array] Array of repositories from which to download DAG files\n ##\n dags:\n enabled: false\n ## Name for repositories can be anything unique and must follow same naming conventions as kubernetes.\n ## Kubernetes resources can have names up to 253 characters long. The characters allowed in names are:\n ## digits (0-9), lower case letters (a-z), -, and .\n ## Example:\n ## - repository: https://github.com/myuser/myrepo\n ## branch: main\n ## name: my-dags\n ## path: /\n ##\n repositories:\n - repository: \"\"\n ## Branch from repository to checkout\n ##\n branch: \"\"\n ## An unique identifier for repository, must be unique for each repository\n ##\n name: \"\"\n ## Path to a folder in the repository containing the dags\n ##\n path: \"\"\n ## Get Plugins files from git repositories.\n ## @param git.plugins.enabled Enable in order to download Plugins files from git repositories.\n ## @param git.plugins.repositories [array] Array of repositories from which to download DAG files\n ##\n plugins:\n enabled: false\n repositories:\n - repository: \"\"\n ## Branch from repository to checkout\n ##\n branch: \"\"\n ## An unique identifier for repository, must be unique for each repository\n ##\n name: \"\"\n ## Path to a folder in the repository containing the plugins\n ##\n path: \"\"\n ## Properties for the Clone init container\n ## @param git.clone.command Override cmd\n ## @param git.clone.args Override args\n ## @param git.clone.extraVolumeMounts Add extra volume mounts\n ## @param git.clone.extraEnvVars Add extra environment variables\n ## @param git.clone.extraEnvVarsCM ConfigMap with extra environment variables\n ## @param git.clone.extraEnvVarsSecret Secret with extra environment variables\n ## @param git.clone.resources Clone init container resource requests and limits\n ##\n clone:\n command: []\n args: []\n extraVolumeMounts: []\n extraEnvVars: []\n extraEnvVarsCM: \"\"\n extraEnvVarsSecret: \"\"\n ## Clone init container resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ##\n resources: {}\n ## Properties for the Sync sidecar container\n ## @param git.sync.interval Interval in seconds to pull the git repository containing the plugins and/or DAG files\n ## @param git.sync.command Override cmd\n ## @param git.sync.args Override args\n ## @param git.sync.extraVolumeMounts Add extra volume mounts\n ## @param git.sync.extraEnvVars Add extra environment variables\n ## @param git.sync.extraEnvVarsCM ConfigMap with extra environment variables\n ## @param git.sync.extraEnvVarsSecret Secret with extra environment variables\n ## @param git.sync.resources Sync sidecar container resource requests and limits\n ##\n sync:\n interval: 60\n command: []\n args: []\n extraVolumeMounts: []\n extraEnvVars: []\n extraEnvVarsCM: \"\"\n extraEnvVarsSecret: \"\"\n ## Sync sidecar container resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ##\n resources: {}\n\n## @section Airflow ldap parameters\n\n## LDAP configuration\n## @param ldap.enabled Enable LDAP authentication\n## @param ldap.uri Server URI, eg. ldap://ldap_server:389\n## DEPRECATED ldap.base It will be removed in a future release, please use 'ldap.basedn' instead\n## @param ldap.basedn Base of the search, eg. ou=example,o=org.\n## DEPRECATED ldap.uidField It will be removed in a future release,, please use 'ldap.searchAttribute' instead\n## @param ldap.searchAttribute if doing an indirect bind to ldap, this is the field that matches the username when searching for the account to bind to\n## @param ldap.binddn DN of the account used to search in the LDAP server.\n## @param ldap.bindpw Bind Password\n## @param ldap.userRegistration Set to True to enable user self registration\n## @param ldap.userRegistrationRole Set role name to be assign when a user registers himself. This role must already exist. Mandatory when using ldap.userRegistration\n## @param ldap.rolesMapping mapping from LDAP DN to a list of roles\n## @param ldap.rolesSyncAtLogin replace ALL the user's roles each login, or only on registration\n##\nldap:\n enabled: false\n uri: \"ldap://ldap_server:389\"\n basedn: \"dc=example,dc=org\"\n searchAttribute: \"cn\"\n binddn: \"cn=admin,dc=example,dc=org\"\n bindpw: \"\"\n userRegistration: 'True'\n userRegistrationRole: \"Public\"\n rolesMapping: '{ \"cn=All,ou=Groups,dc=example,dc=org\": [\"User\"], \"cn=Admins,ou=Groups,dc=example,dc=org\": [\"Admin\"], }'\n rolesSyncAtLogin: 'True'\n\n ## SSL/TLS parameters for LDAP\n ## @param ldap.tls.enabled Enabled TLS/SSL for LDAP, you must include the CA file.\n ## @param ldap.tls.allowSelfSigned Allow to use self signed certificates\n ## DEPRECATED ldap.tls.CAcertificateSecret It will be removed in a future release, please use ldap.tls.certificatesSecret instead\n ## @param ldap.tls.certificatesSecret Name of the existing secret containing the certificate CA file that will be used by ldap client\n ## @param ldap.tls.certificatesMountPath Where LDAP certifcates are mounted.\n ## DEPRECATED ldap.tls.CAcertificateFilename It will be removed in a future release, please use ldap.tls.CAFilename instead\n ## @param ldap.tls.CAFilename LDAP CA cert filename\n ##\n tls:\n enabled: false\n allowSelfSigned: true\n certificatesSecret: \"\"\n certificatesMountPath: /opt/bitnami/airflow/conf/certs\n CAFilename: \"\"\n\n## @section Traffic Exposure Parameters\n\n## Airflow service parameters\n##\nservice:\n ## @param service.type Airflow service type\n ##\n type: ClusterIP\n ## @param service.ports.http Airflow service HTTP port\n ##\n ports:\n http: 8080\n ## Node ports to expose\n ## @param service.nodePorts.http Node port for HTTP\n ## NOTE: choose port between <30000-32767>\n ##\n nodePorts:\n http: \"\"\n ## @param service.sessionAffinity Control where client requests go, to the same pod or round-robin\n ## Values: ClientIP or None\n ## ref: https://kubernetes.io/docs/user-guide/services/\n ##\n sessionAffinity: None\n ## @param service.sessionAffinityConfig Additional settings for the sessionAffinity\n ## sessionAffinityConfig:\n ## clientIP:\n ## timeoutSeconds: 300\n ##\n sessionAffinityConfig: {}\n ## @param service.clusterIP Airflow service Cluster IP\n ## e.g.:\n ## clusterIP: None\n ##\n clusterIP: \"\"\n ## @param service.loadBalancerIP Airflow service Load Balancer IP\n ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-loadbalancer\n ##\n loadBalancerIP: \"\"\n ## @param service.loadBalancerSourceRanges Airflow service Load Balancer sources\n ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service\n ## e.g:\n ## loadBalancerSourceRanges:\n ## - 10.10.10.0/24\n ##\n loadBalancerSourceRanges: []\n ## @param service.externalTrafficPolicy Airflow service external traffic policy\n ## ref https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip\n ##\n externalTrafficPolicy: Cluster\n ## @param service.annotations Additional custom annotations for Airflow service\n ##\n annotations: {}\n ## @param service.extraPorts Extra port to expose on Airflow service\n ##\n extraPorts: []\n\n## Airflow ingress parameters\n## ref: https://kubernetes.io/docs/user-guide/ingress/\n##\ningress:\n ## @param ingress.enabled Enable ingress record generation for Airflow\n ##\n enabled: false\n ## @param ingress.ingressClassName IngressClass that will be be used to implement the Ingress (Kubernetes 1.18+)\n ## This is supported in Kubernetes 1.18+ and required if you have more than one IngressClass marked as the default for your cluster .\n ## ref: https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/\n ##\n ingressClassName: \"\"\n ## @param ingress.pathType Ingress path type\n ##\n pathType: ImplementationSpecific\n ## @param ingress.apiVersion Force Ingress API version (automatically detected if not set)\n ##\n apiVersion: \"\"\n ## @param ingress.hostname Default host for the ingress record\n ##\n hostname: airflow.local\n ## @param ingress.path Default path for the ingress record\n ## NOTE: You may need to set this to '/*' in order to use this with ALB ingress controllers\n ##\n path: /\n ## @param ingress.annotations [object] Additional annotations for the Ingress resource. To enable certificate autogeneration, place here your cert-manager annotations.\n ## Use this parameter to set the required annotations for cert-manager, see\n ## ref: https://cert-manager.io/docs/usage/ingress/#supported-annotations\n ## e.g:\n ## annotations:\n ## kubernetes.io/ingress.class: nginx\n ## cert-manager.io/cluster-issuer: cluster-issuer-name\n ##\n annotations: {}\n ## @param ingress.tls Enable TLS configuration for the host defined at `ingress.hostname` parameter\n ## TLS certificates will be retrieved from a TLS secret with name: `{{- printf \"%s-tls\" .Values.ingress.hostname }}`\n ## You can:\n ## - Use the `ingress.secrets` parameter to create this TLS secret\n ## - Rely on cert-manager to create it by setting the corresponding annotations\n ## - Rely on Helm to create self-signed certificates by setting `ingress.selfSigned=true`\n ##\n tls: false\n ## @param ingress.selfSigned Create a TLS secret for this ingress record using self-signed certificates generated by Helm\n ##\n selfSigned: false\n ## @param ingress.extraHosts An array with additional hostname(s) to be covered with the ingress record\n ## e.g:\n ## extraHosts:\n ## - name: airflow.local\n ## path: /\n ##\n extraHosts: []\n ## @param ingress.extraPaths An array with additional arbitrary paths that may need to be added to the ingress under the main host\n ## e.g:\n ## extraPaths:\n ## - path: /*\n ## backend:\n ## serviceName: ssl-redirect\n ## servicePort: use-annotation\n ##\n extraPaths: []\n ## @param ingress.extraTls TLS configuration for additional hostname(s) to be covered with this ingress record\n ## ref: https://kubernetes.io/docs/concepts/services-networking/ingress/#tls\n ## e.g:\n ## extraTls:\n ## - hosts:\n ## - airflow.local\n ## secretName: airflow.local-tls\n ##\n extraTls: []\n ## @param ingress.secrets Custom TLS certificates as secrets\n ## NOTE: 'key' and 'certificate' are expected in PEM format\n ## NOTE: 'name' should line up with a 'secretName' set further up\n ## If it is not set and you're using cert-manager, this is unneeded, as it will create a secret for you with valid certificates\n ## If it is not set and you're NOT using cert-manager either, self-signed certificates will be created valid for 365 days\n ## It is also possible to create and manage the certificates outside of this helm chart\n ## Please see README.md for more information\n ## e.g:\n ## secrets:\n ## - name: airflow.local-tls\n ## key: |-\n ## -----BEGIN RSA PRIVATE KEY-----\n ## ...\n ## -----END RSA PRIVATE KEY-----\n ## certificate: |-\n ## -----BEGIN CERTIFICATE-----\n ## ...\n ## -----END CERTIFICATE-----\n ##\n secrets: []\n ## @param ingress.extraRules Additional rules to be covered with this ingress record\n ## ref: https://kubernetes.io/docs/concepts/services-networking/ingress/#ingress-rules\n ## e.g:\n ## extraRules:\n ## - host: example.local\n ## http:\n ## path: /\n ## backend:\n ## service:\n ## name: example-svc\n ## port:\n ## name: http\n ##\n extraRules: []\n\n## @section Other Parameters\n\n## Service account for Airflow pods to use.\n## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/\n##\nserviceAccount:\n ## @param serviceAccount.create Enable creation of ServiceAccount for Airflow pods\n ##\n create: false\n ## @param serviceAccount.name The name of the ServiceAccount to use.\n ## If not set and create is true, a name is generated using the common.names.fullname template\n ##\n name: \"\"\n ## @param serviceAccount.automountServiceAccountToken Allows auto mount of ServiceAccountToken on the serviceAccount created\n ## Can be set to false if pods using this serviceAccount do not need to use K8s API\n ##\n automountServiceAccountToken: true\n ## @param serviceAccount.annotations Additional custom annotations for the ServiceAccount\n ##\n annotations: {}\n## Role Based Access\n## Ref: https://kubernetes.io/docs/admin/authorization/rbac/\n## @param rbac.create Create Role and RoleBinding\n##\nrbac:\n create: false\n ## @param rbac.rules Custom RBAC rules to set\n ## e.g:\n ## rules:\n ## - apiGroups:\n ## - \"\"\n ## resources:\n ## - pods\n ## verbs:\n ## - get\n ## - list\n ##\n rules: []\n\n## @section Airflow metrics parameters\n\nmetrics:\n ## @param metrics.enabled Whether or not to create a standalone Airflow exporter to expose Airflow metrics\n ##\n enabled: false\n ## Bitnami Airflow exporter image\n ## ref: https://hub.docker.com/r/bitnami/airflow-exporter/tags/\n ## @param metrics.image.registry Airflow exporter image registry\n ## @param metrics.image.repository Airflow exporter image repository\n ## @param metrics.image.tag Airflow exporter image tag (immutable tags are recommended)\n ## @param metrics.image.digest Airflow exporter image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param metrics.image.pullPolicy Airflow exporter image pull policy\n ## @param metrics.image.pullSecrets Airflow exporter image pull secrets\n ##\n image:\n registry: docker.io\n repository: bitnami/airflow-exporter\n tag: 0.20220314.0-debian-11-r57\n digest: \"\"\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## @param metrics.extraEnvVars Array with extra environment variables to add Airflow exporter pods\n ##\n extraEnvVars: []\n ## @param metrics.extraEnvVarsCM ConfigMap containing extra environment variables for Airflow exporter pods\n ##\n extraEnvVarsCM: \"\"\n ## @param metrics.extraEnvVarsSecret Secret containing extra environment variables (in case of sensitive data) for Airflow exporter pods\n ##\n extraEnvVarsSecret: \"\"\n ## @param metrics.containerPorts.http Airflow exporter metrics container port\n ##\n containerPorts:\n http: 9112\n ## Airflow exporter resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ## @param metrics.resources.limits The resources limits for the container\n ## @param metrics.resources.requests The requested resources for the container\n ##\n resources:\n limits: {}\n requests: {}\n ## Airflow exporter pods' Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod\n ## @param metrics.podSecurityContext.enabled Enable security context for the pods\n ## @param metrics.podSecurityContext.fsGroup Set Airflow exporter pod's Security Context fsGroup\n ##\n podSecurityContext:\n enabled: true\n fsGroup: 1001\n ## Airflow exporter containers' Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container\n ## @param metrics.containerSecurityContext.enabled Enable Airflow exporter containers' Security Context\n ## @param metrics.containerSecurityContext.runAsUser Set Airflow exporter containers' Security Context runAsUser\n ## @param metrics.containerSecurityContext.runAsNonRoot Set Airflow exporter containers' Security Context runAsNonRoot\n ## e.g:\n ## containerSecurityContext:\n ## enabled: true\n ## capabilities:\n ## drop: [\"NET_RAW\"]\n ## readOnlyRootFilesystem: true\n ##\n containerSecurityContext:\n enabled: true\n runAsUser: 1001\n runAsNonRoot: true\n ## @param metrics.lifecycleHooks for the Airflow exporter container(s) to automate configuration before or after startup\n ##\n lifecycleHooks: {}\n ## @param metrics.hostAliases Airflow exporter pods host aliases\n ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/\n ##\n hostAliases: []\n ## @param metrics.podLabels Extra labels for Airflow exporter pods\n ## Ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/\n ##\n podLabels: {}\n ## @param metrics.podAnnotations Extra annotations for Airflow exporter pods\n ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/\n ##\n podAnnotations: {}\n ## @param metrics.podAffinityPreset Pod affinity preset. Ignored if `metrics.affinity` is set. Allowed values: `soft` or `hard`\n ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAffinityPreset: \"\"\n ## @param metrics.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `metrics.affinity` is set. Allowed values: `soft` or `hard`\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAntiAffinityPreset: soft\n ## Node metrics.affinity preset\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity\n ##\n nodeAffinityPreset:\n ## @param metrics.nodeAffinityPreset.type Node affinity preset type. Ignored if `metrics.affinity` is set. Allowed values: `soft` or `hard`\n ##\n type: \"\"\n ## @param metrics.nodeAffinityPreset.key Node label key to match Ignored if `metrics.affinity` is set.\n ## E.g.\n ## key: \"kubernetes.io/e2e-az-name\"\n ##\n key: \"\"\n ## @param metrics.nodeAffinityPreset.values Node label values to match. Ignored if `metrics.affinity` is set.\n ## E.g.\n ## values:\n ## - e2e-az1\n ## - e2e-az2\n ##\n values: []\n ## @param metrics.affinity Affinity for pod assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity\n ## Note: metrics.podAffinityPreset, metrics.podAntiAffinityPreset, and metrics.nodeAffinityPreset will be ignored when it's set\n ##\n affinity: {}\n ## @param metrics.nodeSelector Node labels for pod assignment\n ## Ref: https://kubernetes.io/docs/user-guide/node-selection/\n ##\n nodeSelector: {}\n ## @param metrics.tolerations Tolerations for pod assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n ##\n tolerations: []\n ## @param metrics.schedulerName Name of the k8s scheduler (other than default) for Airflow exporter\n ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/\n ##\n schedulerName: \"\"\n ## Airflow exporter service configuration\n ##\n service:\n ## @param metrics.service.ports.http Airflow exporter metrics service port\n ##\n ports:\n http: 9112\n ## @param metrics.service.clusterIP Static clusterIP or None for headless services\n ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#choosing-your-own-ip-address\n ##\n clusterIP: \"\"\n ## @param metrics.service.sessionAffinity Control where client requests go, to the same pod or round-robin\n ## Values: ClientIP or None\n ## ref: https://kubernetes.io/docs/user-guide/services/\n ##\n sessionAffinity: None\n ## @param metrics.service.annotations [object] Annotations for the Airflow exporter service\n ##\n annotations:\n prometheus.io/scrape: \"true\"\n prometheus.io/port: \"{{ .Values.metrics.service.ports.http }}\"\n ## Prometheus Operator ServiceMonitor configuration\n ##\n serviceMonitor:\n ## @param metrics.serviceMonitor.enabled if `true`, creates a Prometheus Operator ServiceMonitor (requires `metrics.enabled` to be `true`)\n ##\n enabled: false\n ## @param metrics.serviceMonitor.namespace Namespace in which Prometheus is running\n ##\n namespace: \"\"\n ## @param metrics.serviceMonitor.interval Interval at which metrics should be scraped\n ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint\n ##\n interval: \"\"\n ## @param metrics.serviceMonitor.scrapeTimeout Timeout after which the scrape is ended\n ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint\n ##\n scrapeTimeout: \"\"\n ## @param metrics.serviceMonitor.labels Additional labels that can be used so ServiceMonitor will be discovered by Prometheus\n ##\n labels: {}\n ## @param metrics.serviceMonitor.selector Prometheus instance selector labels\n ## ref: https://github.com/bitnami/charts/tree/main/bitnami/prometheus-operator#prometheus-configuration\n ##\n selector: {}\n ## @param metrics.serviceMonitor.relabelings RelabelConfigs to apply to samples before scraping\n ##\n relabelings: []\n ## @param metrics.serviceMonitor.metricRelabelings MetricRelabelConfigs to apply to samples before ingestion\n ##\n metricRelabelings: []\n ## @param metrics.serviceMonitor.honorLabels Specify honorLabels parameter to add the scrape endpoint\n ##\n honorLabels: false\n ## @param metrics.serviceMonitor.jobLabel The name of the label on the target service to use as the job name in prometheus.\n ##\n jobLabel: \"\"\n\n## @section Airflow database parameters\n\n## PostgreSQL chart configuration\n## ref: https://github.com/bitnami/charts/blob/main/bitnami/postgresql/values.yaml\n## @param postgresql.enabled Switch to enable or disable the PostgreSQL helm chart\n## @param postgresql.auth.enablePostgresUser Assign a password to the \"postgres\" admin user. Otherwise, git access will be blocked for this user\n## @param postgresql.auth.username Name for a custom user to create\n## @param postgresql.auth.password Password for the custom user to create\n## @param postgresql.auth.database Name for a custom database to create\n## @param postgresql.auth.existingSecret Name of existing secret to use for PostgreSQL credentials\n## @param postgresql.architecture PostgreSQL architecture (`standalone` or `replication`)\n##\npostgresql:\n enabled: true\n auth:\n enablePostgresUser: false\n username: bn_airflow\n password: \"\"\n database: bitnami_airflow\n existingSecret: \"\"\n architecture: standalone\n## External PostgreSQL configuration\n## All of these values are only used when postgresql.enabled is set to false\n## @param externalDatabase.host Database host\n## @param externalDatabase.port Database port number\n## @param externalDatabase.user Non-root username for Airflow\n## @param externalDatabase.password Password for the non-root username for Airflow\n## @param externalDatabase.database Airflow database name\n## @param externalDatabase.existingSecret Name of an existing secret resource containing the database credentials\n## @param externalDatabase.existingSecretPasswordKey Name of an existing secret key containing the database credentials\n##\nexternalDatabase:\n host: localhost\n port: 5432\n user: bn_airflow\n database: bitnami_airflow\n password: \"\"\n existingSecret: \"\"\n existingSecretPasswordKey: \"\"\n\n## Redis® chart configuration\n## ref: https://github.com/bitnami/charts/blob/main/bitnami/redis/values.yaml\n## @param redis.enabled Switch to enable or disable the Redis® helm\n## @param redis.auth.enabled Enable password authentication\n## @param redis.auth.password Redis® password\n## @param redis.auth.existingSecret The name of an existing secret with Redis® credentials\n## @param redis.architecture Redis® architecture. Allowed values: `standalone` or `replication`\n##\nredis:\n enabled: true\n auth:\n enabled: true\n ## Redis® password (both master and slave). Defaults to a random 10-character alphanumeric string if not set and auth.enabled is true.\n ## It should always be set using the password value or in the existingSecret to avoid issues\n ## with Airflow.\n ## The password value is ignored if existingSecret is set\n password: \"\"\n existingSecret: \"\"\n architecture: standalone\n\n## External Redis® configuration\n## All of these values are only used when redis.enabled is set to false\n## @param externalRedis.host Redis® host\n## @param externalRedis.port Redis® port number\n## @param externalRedis.username Redis® username\n## @param externalRedis.password Redis® password\n## @param externalRedis.existingSecret Name of an existing secret resource containing the Redis&trade credentials\n## @param externalRedis.existingSecretPasswordKey Name of an existing secret key containing the Redis&trade credentials\n##\nexternalRedis:\n host: localhost\n port: 6379\n ## Most Redis® implementations do not require a username\n ## to authenticate and it should be enough with the password\n username: \"\"\n password: \"\"\n existingSecret: \"\"\n existingSecretPasswordKey: \"\"\n", + ValuesOverrideYaml: "## @section Global parameters\n## Global Docker image parameters\n## Please, note that this will override the image parameters, including dependencies, configured to use the global value\n## Current available global Docker image parameters: imageRegistry, imagePullSecrets and storageClass\n\n## @param global.imageRegistry Global Docker image registry\n## @param global.imagePullSecrets Global Docker registry secret names as an array\n## @param global.storageClass Global StorageClass for Persistent Volume(s)\n##\nglobal:\n imageRegistry: \"\"\n ## E.g.\n ## imagePullSecrets:\n ## - myRegistryKeySecretName\n ##\n imagePullSecrets: []\n storageClass: \"\"\n\n## @section Common parameters\n\n## @param kubeVersion Override Kubernetes version\n##\nkubeVersion: \"\"\n## @param nameOverride String to partially override common.names.fullname template (will maintain the release name)\n##\nnameOverride: \"\"\n## @param fullnameOverride String to fully override common.names.fullname template\n##\nfullnameOverride: \"\"\n## @param clusterDomain Kubernetes Cluster Domain\n##\nclusterDomain: cluster.local\n## @param extraDeploy Extra objects to deploy (evaluated as a template)\n##\nextraDeploy: []\n## @param commonLabels Add labels to all the deployed resources\n##\ncommonLabels: {}\n## @param commonAnnotations Add annotations to all the deployed resources\n##\ncommonAnnotations: {}\n## Enable diagnostic mode in the deployment(s)/statefulset(s)\n##\ndiagnosticMode:\n ## @param diagnosticMode.enabled Enable diagnostic mode (all probes will be disabled and the command will be overridden)\n ##\n enabled: false\n ## @param diagnosticMode.command Command to override all containers in the the deployment(s)/statefulset(s)\n ##\n command:\n - sleep\n ## @param diagnosticMode.args Args to override all containers in the the deployment(s)/statefulset(s)\n ##\n args:\n - infinity\n\n## @section Airflow common parameters\n\n## Authentication parameters\n## ref: https://github.com/bitnami/containers/tree/main/bitnami/airflow#environment-variables\n##\nauth:\n ## @param auth.username Username to access web UI\n ##\n username: user\n ## @param auth.password Password to access web UI\n ##\n password: \"\"\n ## @param auth.fernetKey Fernet key to secure connections\n ## ref: https://airflow.readthedocs.io/en/stable/howto/secure-connections.html\n ## ref: https://bcb.github.io/airflow/fernet-key\n ##\n fernetKey: \"\"\n ## @param auth.secretKey Secret key to run your flask app\n ## ref: https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.html#secret-key\n ##\n secretKey: \"\"\n ## @param auth.existingSecret Name of an existing secret to use for Airflow credentials\n ## `auth.password`, `auth.fernetKey`, and `auth.secretKey` will be ignored and picked up from this secret\n ## The secret must contain the keys `airflow-password`, `airflow-fernet-key` and `airflow-secret-key'\n ## The value is evaluated as a template\n ##\n existingSecret: \"\"\n## @param executor Airflow executor. Allowed values: `SequentialExecutor`, `LocalExecutor`, `CeleryExecutor`, `KubernetesExecutor`, `CeleryKubernetesExecutor` and `LocalKubernetesExecutor`\n## ref: http://airflow.apache.org/docs/stable/executor/index.html\n##\nexecutor: CeleryExecutor\n## @param loadExamples Switch to load some Airflow examples\n##\nloadExamples: false\n## @param configuration Specify content for Airflow config file (auto-generated based on other env. vars otherwise)\n## e.g:\n## configuration: |-\n## [core]\n## dags_folder=/opt/bitnami/airflow/dags\n## ...\n##\nconfiguration: \"\"\n## @param existingConfigmap Name of an existing ConfigMap with the Airflow config file\n##\nexistingConfigmap: \"\"\n## Load custom DAGs from a ConfigMap\n## Note: an init container will be used to prepare the DAGs available in the ConfigMap to be consumed by Airflow containers\n##\ndags:\n ## @param dags.existingConfigmap Name of an existing ConfigMap with all the DAGs files you want to load in Airflow\n ##\n existingConfigmap: \"\"\n ## Bitnami Shell image\n ## ref: https://hub.docker.com/r/bitnami/bitnami-shell/tags/\n ## @param dags.image.registry Init container load-dags image registry\n ## @param dags.image.repository Init container load-dags image repository\n ## @param dags.image.tag Init container load-dags image tag (immutable tags are recommended)\n ## @param dags.image.digest Init container load-dags image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param dags.image.pullPolicy Init container load-dags image pull policy\n ## @param dags.image.pullSecrets Init container load-dags image pull secrets\n ##\n image:\n registry: docker.io\n repository: bitnami/bitnami-shell\n tag: 11-debian-11-r50\n digest: \"\"\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n## @param extraEnvVars Add extra environment variables for all the Airflow pods\n##\nextraEnvVars: []\n## @param extraEnvVarsCM ConfigMap with extra environment variables for all the Airflow pods\n##\nextraEnvVarsCM: \"\"\n## @param extraEnvVarsSecret Secret with extra environment variables for all the Airflow pods\n##\nextraEnvVarsSecret: \"\"\n## @param extraEnvVarsSecrets List of secrets with extra environment variables for all the Airflow pods\n##\nextraEnvVarsSecrets: []\n## @param sidecars Add additional sidecar containers to all the Airflow pods\n## Example:\n## sidecars:\n## - name: your-image-name\n## image: your-image\n## imagePullPolicy: Always\n## ports:\n## - name: portname\n## containerPort: 1234\n##\nsidecars: []\n## @param initContainers Add additional init containers to all the Airflow pods\n## Example:\n## initContainers:\n## - name: your-image-name\n## image: your-image\n## imagePullPolicy: Always\n## ports:\n## - name: portname\n## containerPort: 1234\n##\ninitContainers: []\n## @param extraVolumeMounts Optionally specify extra list of additional volumeMounts for all the Airflow pods\n##\nextraVolumeMounts: []\n## @param extraVolumes Optionally specify extra list of additional volumes for the all the Airflow pods\n##\nextraVolumes: []\n\n## @section Airflow web parameters\n\nweb:\n ## Bitnami Airflow image version\n ## ref: https://hub.docker.com/r/bitnami/airflow/tags/\n ## @param web.image.registry Airflow image registry\n ## @param web.image.repository Airflow image repository\n ## @param web.image.tag Airflow image tag (immutable tags are recommended)\n ## @param web.image.digest Airflow image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param web.image.pullPolicy Airflow image pull policy\n ## @param web.image.pullSecrets Airflow image pull secrets\n ## @param web.image.debug Enable image debug mode\n image:\n registry: docker.io\n repository: bitnami/airflow\n tag: 2.4.2-debian-11-r6\n digest: \"\"\n ## Specify a imagePullPolicy\n ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'\n ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images\n ##\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## Set to true if you would like to see extra information on logs\n ##\n debug: false\n ## @param web.baseUrl URL used to access to Airflow web ui\n ##\n baseUrl: \"\"\n ## @param web.existingConfigmap Name of an existing config map containing the Airflow web config file\n ##\n existingConfigmap: \"\"\n ## @param web.command Override default container command (useful when using custom images)\n ##\n command: []\n ## @param web.args Override default container args (useful when using custom images)\n ##\n args: []\n ## @param web.extraEnvVars Array with extra environment variables to add Airflow web pods\n ##\n extraEnvVars: []\n ## @param web.extraEnvVarsCM ConfigMap containing extra environment variables for Airflow web pods\n ##\n extraEnvVarsCM: \"\"\n ## @param web.extraEnvVarsSecret Secret containing extra environment variables (in case of sensitive data) for Airflow web pods\n ##\n extraEnvVarsSecret: \"\"\n ## @param web.extraEnvVarsSecrets List of secrets with extra environment variables for Airflow web pods\n ##\n extraEnvVarsSecrets: []\n ## @param web.containerPorts.http Airflow web HTTP container port\n ##\n containerPorts:\n http: 8080\n ## @param web.replicaCount Number of Airflow web replicas\n ##\n replicaCount: 1\n ## Configure extra options for Airflow web containers' liveness, readiness and startup probes\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes\n ## @param web.livenessProbe.enabled Enable livenessProbe on Airflow web containers\n ## @param web.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe\n ## @param web.livenessProbe.periodSeconds Period seconds for livenessProbe\n ## @param web.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe\n ## @param web.livenessProbe.failureThreshold Failure threshold for livenessProbe\n ## @param web.livenessProbe.successThreshold Success threshold for livenessProbe\n ##\n livenessProbe:\n enabled: true\n initialDelaySeconds: 180\n periodSeconds: 20\n timeoutSeconds: 5\n failureThreshold: 6\n successThreshold: 1\n ## @param web.readinessProbe.enabled Enable readinessProbe on Airflow web containers\n ## @param web.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe\n ## @param web.readinessProbe.periodSeconds Period seconds for readinessProbe\n ## @param web.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe\n ## @param web.readinessProbe.failureThreshold Failure threshold for readinessProbe\n ## @param web.readinessProbe.successThreshold Success threshold for readinessProbe\n ##\n readinessProbe:\n enabled: true\n initialDelaySeconds: 30\n periodSeconds: 10\n timeoutSeconds: 5\n failureThreshold: 6\n successThreshold: 1\n ## @param web.startupProbe.enabled Enable startupProbe on Airflow web containers\n ## @param web.startupProbe.initialDelaySeconds Initial delay seconds for startupProbe\n ## @param web.startupProbe.periodSeconds Period seconds for startupProbe\n ## @param web.startupProbe.timeoutSeconds Timeout seconds for startupProbe\n ## @param web.startupProbe.failureThreshold Failure threshold for startupProbe\n ## @param web.startupProbe.successThreshold Success threshold for startupProbe\n ##\n startupProbe:\n enabled: false\n initialDelaySeconds: 60\n periodSeconds: 10\n timeoutSeconds: 1\n failureThreshold: 15\n successThreshold: 1\n ## @param web.customLivenessProbe Custom livenessProbe that overrides the default one\n ##\n customLivenessProbe: {}\n ## @param web.customReadinessProbe Custom readinessProbe that overrides the default one\n ##\n customReadinessProbe: {}\n ## @param web.customStartupProbe Custom startupProbe that overrides the default one\n ##\n customStartupProbe: {}\n ## Airflow web resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ## @param web.resources.limits The resources limits for the Airflow web containers\n ## @param web.resources.requests The requested resources for the Airflow web containers\n ##\n resources:\n limits: {}\n requests: {}\n ## Configure Airflow web pods Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod\n ## @param web.podSecurityContext.enabled Enabled Airflow web pods' Security Context\n ## @param web.podSecurityContext.fsGroup Set Airflow web pod's Security Context fsGroup\n ##\n podSecurityContext:\n enabled: true\n fsGroup: 1001\n ## Configure Airflow web containers (only main one) Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container\n ## @param web.containerSecurityContext.enabled Enabled Airflow web containers' Security Context\n ## @param web.containerSecurityContext.runAsUser Set Airflow web containers' Security Context runAsUser\n ## @param web.containerSecurityContext.runAsNonRoot Set Airflow web containers' Security Context runAsNonRoot\n ##\n containerSecurityContext:\n enabled: true\n runAsUser: 1001\n runAsNonRoot: true\n ## @param web.lifecycleHooks for the Airflow web container(s) to automate configuration before or after startup\n ##\n lifecycleHooks: {}\n ## @param web.hostAliases Deployment pod host aliases\n ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/\n ##\n hostAliases: []\n ## @param web.podLabels Add extra labels to the Airflow web pods\n ##\n podLabels: {}\n ## @param web.podAnnotations Add extra annotations to the Airflow web pods\n ##\n podAnnotations: {}\n ## @param web.affinity Affinity for Airflow web pods assignment (evaluated as a template)\n ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity\n ## Note: `web.podAffinityPreset`, `web.podAntiAffinityPreset`, and `web.nodeAffinityPreset` will be ignored when it's set\n ##\n affinity: {}\n ## Node affinity preset\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity\n ## @param web.nodeAffinityPreset.key Node label key to match. Ignored if `web.affinity` is set.\n ## @param web.nodeAffinityPreset.type Node affinity preset type. Ignored if `web.affinity` is set. Allowed values: `soft` or `hard`\n ## @param web.nodeAffinityPreset.values Node label values to match. Ignored if `web.affinity` is set.\n ##\n nodeAffinityPreset:\n ## e.g:\n ## key: \"kubernetes.io/e2e-az-name\"\n ##\n key: \"\"\n type: \"\"\n ## e.g:\n ## values:\n ## - e2e-az1\n ## - e2e-az2\n ##\n values: []\n ## @param web.nodeSelector Node labels for Airflow web pods assignment\n ## Ref: https://kubernetes.io/docs/user-guide/node-selection/\n ##\n nodeSelector: {}\n ## @param web.podAffinityPreset Pod affinity preset. Ignored if `web.affinity` is set. Allowed values: `soft` or `hard`.\n ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAffinityPreset: \"\"\n ## @param web.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `web.affinity` is set. Allowed values: `soft` or `hard`.\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAntiAffinityPreset: soft\n ## @param web.tolerations Tolerations for Airflow web pods assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n ##\n tolerations: []\n ## @param web.topologySpreadConstraints Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template\n ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods\n ##\n topologySpreadConstraints: []\n ## @param web.priorityClassName Priority Class Name\n ## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass\n ##\n priorityClassName: \"\"\n ## @param web.schedulerName Use an alternate scheduler, e.g. \"stork\".\n ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/\n ##\n schedulerName: \"\"\n ## @param web.terminationGracePeriodSeconds Seconds Airflow web pod needs to terminate gracefully\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods\n ##\n terminationGracePeriodSeconds: \"\"\n ## @param web.updateStrategy.type Airflow web deployment strategy type\n ## @param web.updateStrategy.rollingUpdate Airflow web deployment rolling update configuration parameters\n ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy\n ##\n updateStrategy:\n type: RollingUpdate\n rollingUpdate: {}\n ## @param web.sidecars Add additional sidecar containers to the Airflow web pods\n ## Example:\n ## sidecars:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n sidecars: []\n ## @param web.initContainers Add additional init containers to the Airflow web pods\n ## Example:\n ## initContainers:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n initContainers: []\n ## @param web.extraVolumeMounts Optionally specify extra list of additional volumeMounts for the Airflow web pods\n ##\n extraVolumeMounts: []\n ## @param web.extraVolumes Optionally specify extra list of additional volumes for the Airflow web pods\n ##\n extraVolumes: []\n ## Airflow web Pod Disruption Budget\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/\n ## @param web.pdb.create Deploy a pdb object for the Airflow web pods\n ## @param web.pdb.minAvailable Maximum number/percentage of unavailable Airflow web replicas\n ## @param web.pdb.maxUnavailable Maximum number/percentage of unavailable Airflow web replicas\n ##\n pdb:\n create: false\n minAvailable: 1\n maxUnavailable: \"\"\n\n## @section Airflow scheduler parameters\n\nscheduler:\n ## Bitnami Airflow Scheduler image version\n ## ref: https://hub.docker.com/r/bitnami/airflow-scheduler/tags/\n ## @param scheduler.image.registry Airflow Scheduler image registry\n ## @param scheduler.image.repository Airflow Scheduler image repository\n ## @param scheduler.image.tag Airflow Scheduler image tag (immutable tags are recommended)\n ## @param scheduler.image.digest Airflow Schefuler image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param scheduler.image.pullPolicy Airflow Scheduler image pull policy\n ## @param scheduler.image.pullSecrets Airflow Scheduler image pull secrets\n ## @param scheduler.image.debug Enable image debug mode\n ##\n image:\n registry: docker.io\n repository: bitnami/airflow-scheduler\n tag: 2.4.2-debian-11-r4\n digest: \"\"\n ## Specify a imagePullPolicy\n ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'\n ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images\n ##\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## Set to true if you would like to see extra information on logs\n ##\n debug: false\n ## @param scheduler.replicaCount Number of scheduler replicas\n ##\n replicaCount: 1\n ## @param scheduler.command Override cmd\n ##\n command: []\n ## @param scheduler.args Override args\n ##\n args: []\n ## @param scheduler.extraEnvVars Add extra environment variables\n ##\n extraEnvVars: []\n ## @param scheduler.extraEnvVarsCM ConfigMap with extra environment variables\n ##\n extraEnvVarsCM: \"\"\n ## @param scheduler.extraEnvVarsSecret Secret with extra environment variables\n ##\n extraEnvVarsSecret: \"\"\n ## @param scheduler.extraEnvVarsSecrets List of secrets with extra environment variables for Airflow scheduler pods\n ##\n extraEnvVarsSecrets: []\n ## @param scheduler.customLivenessProbe Custom livenessProbe that overrides the default one\n ##\n customLivenessProbe: {}\n ## @param scheduler.customReadinessProbe Custom readinessProbe that overrides the default one\n ##\n customReadinessProbe: {}\n ## @param scheduler.customStartupProbe Custom startupProbe that overrides the default one\n ##\n customStartupProbe: {}\n ## Airflow scheduler resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ## @param scheduler.resources.limits The resources limits for the Airflow scheduler containers\n ## @param scheduler.resources.requests The requested resources for the Airflow scheduler containers\n ##\n resources:\n limits: {}\n requests: {}\n ## Configure Airflow scheduler pods Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod\n ## @param scheduler.podSecurityContext.enabled Enabled Airflow scheduler pods' Security Context\n ## @param scheduler.podSecurityContext.fsGroup Set Airflow scheduler pod's Security Context fsGroup\n ##\n podSecurityContext:\n enabled: true\n fsGroup: 1001\n ## Configure Airflow scheduler containers (only main one) Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container\n ## @param scheduler.containerSecurityContext.enabled Enabled Airflow scheduler containers' Security Context\n ## @param scheduler.containerSecurityContext.runAsUser Set Airflow scheduler containers' Security Context runAsUser\n ## @param scheduler.containerSecurityContext.runAsNonRoot Set Airflow scheduler containers' Security Context runAsNonRoot\n ##\n containerSecurityContext:\n enabled: true\n runAsUser: 1001\n runAsNonRoot: true\n ## @param scheduler.lifecycleHooks for the Airflow scheduler container(s) to automate configuration before or after startup\n ##\n lifecycleHooks: {}\n ## @param scheduler.hostAliases Deployment pod host aliases\n ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/\n ##\n hostAliases: []\n ## @param scheduler.podLabels Add extra labels to the Airflow scheduler pods\n ##\n podLabels: {}\n ## @param scheduler.podAnnotations Add extra annotations to the Airflow scheduler pods\n ##\n podAnnotations: {}\n ## @param scheduler.affinity Affinity for Airflow scheduler pods assignment (evaluated as a template)\n ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity\n ## Note: `scheduler.podAffinityPreset`, `scheduler.podAntiAffinityPreset`, and `scheduler.nodeAffinityPreset` will be ignored when it's set\n ##\n affinity: {}\n ## Node affinity preset\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity\n ## @param scheduler.nodeAffinityPreset.key Node label key to match. Ignored if `scheduler.affinity` is set.\n ## @param scheduler.nodeAffinityPreset.type Node affinity preset type. Ignored if `scheduler.affinity` is set. Allowed values: `soft` or `hard`\n ## @param scheduler.nodeAffinityPreset.values Node label values to match. Ignored if `scheduler.affinity` is set.\n ##\n nodeAffinityPreset:\n ## e.g:\n ## key: \"kubernetes.io/e2e-az-name\"\n ##\n key: \"\"\n type: \"\"\n ## e.g:\n ## values:\n ## - e2e-az1\n ## - e2e-az2\n ##\n values: []\n ## @param scheduler.nodeSelector Node labels for Airflow scheduler pods assignment\n ## Ref: https://kubernetes.io/docs/user-guide/node-selection/\n ##\n nodeSelector: {}\n ## @param scheduler.podAffinityPreset Pod affinity preset. Ignored if `scheduler.affinity` is set. Allowed values: `soft` or `hard`.\n ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAffinityPreset: \"\"\n ## @param scheduler.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `scheduler.affinity` is set. Allowed values: `soft` or `hard`.\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAntiAffinityPreset: soft\n ## @param scheduler.tolerations Tolerations for Airflow scheduler pods assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n ##\n tolerations: []\n ## @param scheduler.topologySpreadConstraints Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template\n ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods\n ##\n topologySpreadConstraints: []\n ## @param scheduler.priorityClassName Priority Class Name\n ## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass\n ##\n priorityClassName: \"\"\n ## @param scheduler.schedulerName Use an alternate scheduler, e.g. \"stork\".\n ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/\n ##\n schedulerName: \"\"\n ## @param scheduler.terminationGracePeriodSeconds Seconds Airflow scheduler pod needs to terminate gracefully\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods\n ##\n terminationGracePeriodSeconds: \"\"\n ## @param scheduler.updateStrategy.type Airflow scheduler deployment strategy type\n ## @param scheduler.updateStrategy.rollingUpdate Airflow scheduler deployment rolling update configuration parameters\n ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy\n ##\n updateStrategy:\n type: RollingUpdate\n rollingUpdate: {}\n ## @param scheduler.sidecars Add additional sidecar containers to the Airflow scheduler pods\n ## Example:\n ## sidecars:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n sidecars: []\n ## @param scheduler.initContainers Add additional init containers to the Airflow scheduler pods\n ## Example:\n ## initContainers:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n initContainers: []\n ## @param scheduler.extraVolumeMounts Optionally specify extra list of additional volumeMounts for the Airflow scheduler pods\n ##\n extraVolumeMounts: []\n ## @param scheduler.extraVolumes Optionally specify extra list of additional volumes for the Airflow scheduler pods\n ##\n extraVolumes: []\n ## Airflow scheduler Pod Disruption Budget\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/\n ## @param scheduler.pdb.create Deploy a pdb object for the Airflow scheduler pods\n ## @param scheduler.pdb.minAvailable Maximum number/percentage of unavailable Airflow scheduler replicas\n ## @param scheduler.pdb.maxUnavailable Maximum number/percentage of unavailable Airflow scheduler replicas\n ##\n pdb:\n create: false\n minAvailable: 1\n maxUnavailable: \"\"\n\n## @section Airflow worker parameters\n\nworker:\n ## Bitnami Airflow Worker image version\n ## ref: https://hub.docker.com/r/bitnami/airflow-worker/tags/\n ## @param worker.image.registry Airflow Worker image registry\n ## @param worker.image.repository Airflow Worker image repository\n ## @param worker.image.tag Airflow Worker image tag (immutable tags are recommended)\n ## @param worker.image.digest Airflow Worker image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param worker.image.pullPolicy Airflow Worker image pull policy\n ## @param worker.image.pullSecrets Airflow Worker image pull secrets\n ## @param worker.image.debug Enable image debug mode\n ##\n image:\n registry: docker.io\n repository: bitnami/airflow-worker\n tag: 2.4.2-debian-11-r4\n digest: \"\"\n ## Specify a imagePullPolicy\n ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'\n ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images\n ##\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## Set to true if you would like to see extra information on logs\n ##\n debug: false\n ## @param worker.command Override default container command (useful when using custom images)\n ##\n command: []\n ## @param worker.args Override default container args (useful when using custom images)\n ##\n args: []\n ## @param worker.extraEnvVars Array with extra environment variables to add Airflow worker pods\n ##\n extraEnvVars: []\n ## @param worker.extraEnvVarsCM ConfigMap containing extra environment variables for Airflow worker pods\n ##\n extraEnvVarsCM: \"\"\n ## @param worker.extraEnvVarsSecret Secret containing extra environment variables (in case of sensitive data) for Airflow worker pods\n ##\n extraEnvVarsSecret: \"\"\n ## @param worker.extraEnvVarsSecrets List of secrets with extra environment variables for Airflow worker pods\n ##\n extraEnvVarsSecrets: []\n ## @param worker.containerPorts.http Airflow worker HTTP container port\n ##\n containerPorts:\n http: 8793\n ## @param worker.replicaCount Number of Airflow worker replicas\n ##\n replicaCount: 1\n ## Configure extra options for Airflow worker containers' liveness, readiness and startup probes\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes\n ## @param worker.livenessProbe.enabled Enable livenessProbe on Airflow worker containers\n ## @param worker.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe\n ## @param worker.livenessProbe.periodSeconds Period seconds for livenessProbe\n ## @param worker.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe\n ## @param worker.livenessProbe.failureThreshold Failure threshold for livenessProbe\n ## @param worker.livenessProbe.successThreshold Success threshold for livenessProbe\n ##\n livenessProbe:\n enabled: true\n initialDelaySeconds: 180\n periodSeconds: 20\n timeoutSeconds: 5\n failureThreshold: 6\n successThreshold: 1\n ## @param worker.readinessProbe.enabled Enable readinessProbe on Airflow worker containers\n ## @param worker.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe\n ## @param worker.readinessProbe.periodSeconds Period seconds for readinessProbe\n ## @param worker.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe\n ## @param worker.readinessProbe.failureThreshold Failure threshold for readinessProbe\n ## @param worker.readinessProbe.successThreshold Success threshold for readinessProbe\n ##\n readinessProbe:\n enabled: true\n initialDelaySeconds: 30\n periodSeconds: 10\n timeoutSeconds: 5\n failureThreshold: 6\n successThreshold: 1\n ## @param worker.startupProbe.enabled Enable startupProbe on Airflow worker containers\n ## @param worker.startupProbe.initialDelaySeconds Initial delay seconds for startupProbe\n ## @param worker.startupProbe.periodSeconds Period seconds for startupProbe\n ## @param worker.startupProbe.timeoutSeconds Timeout seconds for startupProbe\n ## @param worker.startupProbe.failureThreshold Failure threshold for startupProbe\n ## @param worker.startupProbe.successThreshold Success threshold for startupProbe\n ##\n startupProbe:\n enabled: false\n initialDelaySeconds: 60\n periodSeconds: 10\n timeoutSeconds: 1\n failureThreshold: 15\n successThreshold: 1\n ## @param worker.customLivenessProbe Custom livenessProbe that overrides the default one\n ##\n customLivenessProbe: {}\n ## @param worker.customReadinessProbe Custom readinessProbe that overrides the default one\n ##\n customReadinessProbe: {}\n ## @param worker.customStartupProbe Custom startupProbe that overrides the default one\n ##\n customStartupProbe: {}\n ## Airflow worker resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ## @param worker.resources.limits The resources limits for the Airflow worker containers\n ## @param worker.resources.requests The requested resources for the Airflow worker containers\n ##\n resources:\n limits: {}\n requests: {}\n ## Configure Airflow worker pods Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod\n ## @param worker.podSecurityContext.enabled Enabled Airflow worker pods' Security Context\n ## @param worker.podSecurityContext.fsGroup Set Airflow worker pod's Security Context fsGroup\n ##\n podSecurityContext:\n enabled: true\n fsGroup: 1001\n ## Configure Airflow worker containers (only main one) Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container\n ## @param worker.containerSecurityContext.enabled Enabled Airflow worker containers' Security Context\n ## @param worker.containerSecurityContext.runAsUser Set Airflow worker containers' Security Context runAsUser\n ## @param worker.containerSecurityContext.runAsNonRoot Set Airflow worker containers' Security Context runAsNonRoot\n ##\n containerSecurityContext:\n enabled: true\n runAsUser: 1001\n runAsNonRoot: true\n ## @param worker.lifecycleHooks for the Airflow worker container(s) to automate configuration before or after startup\n ##\n lifecycleHooks: {}\n ## @param worker.hostAliases Deployment pod host aliases\n ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/\n ##\n hostAliases: []\n ## @param worker.podLabels Add extra labels to the Airflow worker pods\n ##\n podLabels: {}\n ## @param worker.podAnnotations Add extra annotations to the Airflow worker pods\n ##\n podAnnotations: {}\n ## @param worker.affinity Affinity for Airflow worker pods assignment (evaluated as a template)\n ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity\n ## Note: `worker.podAffinityPreset`, `worker.podAntiAffinityPreset`, and `worker.nodeAffinityPreset` will be ignored when it's set\n ##\n affinity: {}\n ## Node affinity preset\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity\n ## @param worker.nodeAffinityPreset.key Node label key to match. Ignored if `worker.affinity` is set.\n ## @param worker.nodeAffinityPreset.type Node affinity preset type. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard`\n ## @param worker.nodeAffinityPreset.values Node label values to match. Ignored if `worker.affinity` is set.\n ##\n nodeAffinityPreset:\n ## e.g:\n ## key: \"kubernetes.io/e2e-az-name\"\n ##\n key: \"\"\n type: \"\"\n ## e.g:\n ## values:\n ## - e2e-az1\n ## - e2e-az2\n ##\n values: []\n ## @param worker.nodeSelector Node labels for Airflow worker pods assignment\n ## Ref: https://kubernetes.io/docs/user-guide/node-selection/\n ##\n nodeSelector: {}\n ## @param worker.podAffinityPreset Pod affinity preset. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard`.\n ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAffinityPreset: \"\"\n ## @param worker.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `worker.affinity` is set. Allowed values: `soft` or `hard`.\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAntiAffinityPreset: soft\n ## @param worker.tolerations Tolerations for Airflow worker pods assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n ##\n tolerations: []\n ## @param worker.topologySpreadConstraints Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template\n ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods\n ##\n topologySpreadConstraints: []\n ## @param worker.priorityClassName Priority Class Name\n ## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass\n ##\n priorityClassName: \"\"\n ## @param worker.schedulerName Use an alternate scheduler, e.g. \"stork\".\n ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/\n ##\n schedulerName: \"\"\n ## @param worker.terminationGracePeriodSeconds Seconds Airflow worker pod needs to terminate gracefully\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods\n ##\n terminationGracePeriodSeconds: \"\"\n ## @param worker.updateStrategy.type Airflow worker deployment strategy type\n ## @param worker.updateStrategy.rollingUpdate Airflow worker deployment rolling update configuration parameters\n ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy\n ##\n updateStrategy:\n type: RollingUpdate\n rollingUpdate: {}\n ## @param worker.sidecars Add additional sidecar containers to the Airflow worker pods\n ## Example:\n ## sidecars:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n sidecars: []\n ## @param worker.initContainers Add additional init containers to the Airflow worker pods\n ## Example:\n ## initContainers:\n ## - name: your-image-name\n ## image: your-image\n ## imagePullPolicy: Always\n ## ports:\n ## - name: portname\n ## containerPort: 1234\n ##\n initContainers: []\n ## @param worker.extraVolumeMounts Optionally specify extra list of additional volumeMounts for the Airflow worker pods\n ##\n extraVolumeMounts: []\n ## @param worker.extraVolumes Optionally specify extra list of additional volumes for the Airflow worker pods\n ##\n extraVolumes: []\n ## @param worker.extraVolumeClaimTemplates Optionally specify extra list of volumesClaimTemplates for the Airflow worker statefulset\n ##\n extraVolumeClaimTemplates: []\n ## @param worker.podTemplate Template to replace the default one to be use when `executor=KubernetesExecutor` to create Airflow worker pods\n ##\n podTemplate: {}\n ## Airflow worker Pod Disruption Budget\n ## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/\n ## @param worker.pdb.create Deploy a pdb object for the Airflow worker pods\n ## @param worker.pdb.minAvailable Maximum number/percentage of unavailable Airflow worker replicas\n ## @param worker.pdb.maxUnavailable Maximum number/percentage of unavailable Airflow worker replicas\n ##\n pdb:\n create: false\n minAvailable: 1\n maxUnavailable: \"\"\n ## Enable HorizontalPodAutoscaler for worker pods\n ## ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/\n ## @param worker.autoscaling.enabled Whether enable horizontal pod autoscaler\n ## @param worker.autoscaling.minReplicas Configure a minimum amount of pods\n ## @param worker.autoscaling.maxReplicas Configure a maximum amount of pods\n ## @param worker.autoscaling.targetCPU Define the CPU target to trigger the scaling actions (utilization percentage)\n ## @param worker.autoscaling.targetMemory Define the memory target to trigger the scaling actions (utilization percentage)\n ##\n autoscaling:\n enabled: false\n minReplicas: 1\n maxReplicas: 3\n targetCPU: 80\n targetMemory: 80\n\n## @section Airflow git sync parameters\n\n## Configure Git to pull dags and plugins\n##\ngit:\n ## Bitnami Git image version\n ## ref: https://hub.docker.com/r/bitnami/git/tags/\n ## @param git.image.registry Git image registry\n ## @param git.image.repository Git image repository\n ## @param git.image.tag Git image tag (immutable tags are recommended)\n ## @param git.image.digest Git image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param git.image.pullPolicy Git image pull policy\n ## @param git.image.pullSecrets Git image pull secrets\n ##\n image:\n registry: docker.io\n repository: bitnami/git\n tag: 2.38.1-debian-11-r7\n digest: \"\"\n ## Specify a imagePullPolicy\n ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'\n ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images\n ##\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## Get DAG files from git repositories\n ## @param git.dags.enabled Enable in order to download DAG files from git repositories.\n ## @param git.dags.repositories [array] Array of repositories from which to download DAG files\n ##\n dags:\n enabled: false\n ## Name for repositories can be anything unique and must follow same naming conventions as kubernetes.\n ## Kubernetes resources can have names up to 253 characters long. The characters allowed in names are:\n ## digits (0-9), lower case letters (a-z), -, and .\n ## Example:\n ## - repository: https://github.com/myuser/myrepo\n ## branch: main\n ## name: my-dags\n ## path: /\n ##\n repositories:\n - repository: \"\"\n ## Branch from repository to checkout\n ##\n branch: \"\"\n ## An unique identifier for repository, must be unique for each repository\n ##\n name: \"\"\n ## Path to a folder in the repository containing the dags\n ##\n path: \"\"\n ## Get Plugins files from git repositories.\n ## @param git.plugins.enabled Enable in order to download Plugins files from git repositories.\n ## @param git.plugins.repositories [array] Array of repositories from which to download DAG files\n ##\n plugins:\n enabled: false\n repositories:\n - repository: \"\"\n ## Branch from repository to checkout\n ##\n branch: \"\"\n ## An unique identifier for repository, must be unique for each repository\n ##\n name: \"\"\n ## Path to a folder in the repository containing the plugins\n ##\n path: \"\"\n ## Properties for the Clone init container\n ## @param git.clone.command Override cmd\n ## @param git.clone.args Override args\n ## @param git.clone.extraVolumeMounts Add extra volume mounts\n ## @param git.clone.extraEnvVars Add extra environment variables\n ## @param git.clone.extraEnvVarsCM ConfigMap with extra environment variables\n ## @param git.clone.extraEnvVarsSecret Secret with extra environment variables\n ## @param git.clone.resources Clone init container resource requests and limits\n ##\n clone:\n command: []\n args: []\n extraVolumeMounts: []\n extraEnvVars: []\n extraEnvVarsCM: \"\"\n extraEnvVarsSecret: \"\"\n ## Clone init container resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ##\n resources: {}\n ## Properties for the Sync sidecar container\n ## @param git.sync.interval Interval in seconds to pull the git repository containing the plugins and/or DAG files\n ## @param git.sync.command Override cmd\n ## @param git.sync.args Override args\n ## @param git.sync.extraVolumeMounts Add extra volume mounts\n ## @param git.sync.extraEnvVars Add extra environment variables\n ## @param git.sync.extraEnvVarsCM ConfigMap with extra environment variables\n ## @param git.sync.extraEnvVarsSecret Secret with extra environment variables\n ## @param git.sync.resources Sync sidecar container resource requests and limits\n ##\n sync:\n interval: 60\n command: []\n args: []\n extraVolumeMounts: []\n extraEnvVars: []\n extraEnvVarsCM: \"\"\n extraEnvVarsSecret: \"\"\n ## Sync sidecar container resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ##\n resources: {}\n\n## @section Airflow ldap parameters\n\n## LDAP configuration\n## @param ldap.enabled Enable LDAP authentication\n## @param ldap.uri Server URI, eg. ldap://ldap_server:389\n## DEPRECATED ldap.base It will be removed in a future release, please use 'ldap.basedn' instead\n## @param ldap.basedn Base of the search, eg. ou=example,o=org.\n## DEPRECATED ldap.uidField It will be removed in a future release,, please use 'ldap.searchAttribute' instead\n## @param ldap.searchAttribute if doing an indirect bind to ldap, this is the field that matches the username when searching for the account to bind to\n## @param ldap.binddn DN of the account used to search in the LDAP server.\n## @param ldap.bindpw Bind Password\n## @param ldap.userRegistration Set to True to enable user self registration\n## @param ldap.userRegistrationRole Set role name to be assign when a user registers himself. This role must already exist. Mandatory when using ldap.userRegistration\n## @param ldap.rolesMapping mapping from LDAP DN to a list of roles\n## @param ldap.rolesSyncAtLogin replace ALL the user's roles each login, or only on registration\n##\nldap:\n enabled: false\n uri: \"ldap://ldap_server:389\"\n basedn: \"dc=example,dc=org\"\n searchAttribute: \"cn\"\n binddn: \"cn=admin,dc=example,dc=org\"\n bindpw: \"\"\n userRegistration: 'True'\n userRegistrationRole: \"Public\"\n rolesMapping: '{ \"cn=All,ou=Groups,dc=example,dc=org\": [\"User\"], \"cn=Admins,ou=Groups,dc=example,dc=org\": [\"Admin\"], }'\n rolesSyncAtLogin: 'True'\n\n ## SSL/TLS parameters for LDAP\n ## @param ldap.tls.enabled Enabled TLS/SSL for LDAP, you must include the CA file.\n ## @param ldap.tls.allowSelfSigned Allow to use self signed certificates\n ## DEPRECATED ldap.tls.CAcertificateSecret It will be removed in a future release, please use ldap.tls.certificatesSecret instead\n ## @param ldap.tls.certificatesSecret Name of the existing secret containing the certificate CA file that will be used by ldap client\n ## @param ldap.tls.certificatesMountPath Where LDAP certifcates are mounted.\n ## DEPRECATED ldap.tls.CAcertificateFilename It will be removed in a future release, please use ldap.tls.CAFilename instead\n ## @param ldap.tls.CAFilename LDAP CA cert filename\n ##\n tls:\n enabled: false\n allowSelfSigned: true\n certificatesSecret: \"\"\n certificatesMountPath: /opt/bitnami/airflow/conf/certs\n CAFilename: \"\"\n\n## @section Traffic Exposure Parameters\n\n## Airflow service parameters\n##\nservice:\n ## @param service.type Airflow service type\n ##\n type: ClusterIP\n ## @param service.ports.http Airflow service HTTP port\n ##\n ports:\n http: 8080\n ## Node ports to expose\n ## @param service.nodePorts.http Node port for HTTP\n ## NOTE: choose port between <30000-32767>\n ##\n nodePorts:\n http: \"\"\n ## @param service.sessionAffinity Control where client requests go, to the same pod or round-robin\n ## Values: ClientIP or None\n ## ref: https://kubernetes.io/docs/user-guide/services/\n ##\n sessionAffinity: None\n ## @param service.sessionAffinityConfig Additional settings for the sessionAffinity\n ## sessionAffinityConfig:\n ## clientIP:\n ## timeoutSeconds: 300\n ##\n sessionAffinityConfig: {}\n ## @param service.clusterIP Airflow service Cluster IP\n ## e.g.:\n ## clusterIP: None\n ##\n clusterIP: \"\"\n ## @param service.loadBalancerIP Airflow service Load Balancer IP\n ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-loadbalancer\n ##\n loadBalancerIP: \"\"\n ## @param service.loadBalancerSourceRanges Airflow service Load Balancer sources\n ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service\n ## e.g:\n ## loadBalancerSourceRanges:\n ## - 10.10.10.0/24\n ##\n loadBalancerSourceRanges: []\n ## @param service.externalTrafficPolicy Airflow service external traffic policy\n ## ref https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip\n ##\n externalTrafficPolicy: Cluster\n ## @param service.annotations Additional custom annotations for Airflow service\n ##\n annotations: {}\n ## @param service.extraPorts Extra port to expose on Airflow service\n ##\n extraPorts: []\n\n## Airflow ingress parameters\n## ref: https://kubernetes.io/docs/user-guide/ingress/\n##\ningress:\n ## @param ingress.enabled Enable ingress record generation for Airflow\n ##\n enabled: false\n ## @param ingress.ingressClassName IngressClass that will be be used to implement the Ingress (Kubernetes 1.18+)\n ## This is supported in Kubernetes 1.18+ and required if you have more than one IngressClass marked as the default for your cluster .\n ## ref: https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/\n ##\n ingressClassName: \"\"\n ## @param ingress.pathType Ingress path type\n ##\n pathType: ImplementationSpecific\n ## @param ingress.apiVersion Force Ingress API version (automatically detected if not set)\n ##\n apiVersion: \"\"\n ## @param ingress.hostname Default host for the ingress record\n ##\n hostname: airflow.local\n ## @param ingress.path Default path for the ingress record\n ## NOTE: You may need to set this to '/*' in order to use this with ALB ingress controllers\n ##\n path: /\n ## @param ingress.annotations [object] Additional annotations for the Ingress resource. To enable certificate autogeneration, place here your cert-manager annotations.\n ## Use this parameter to set the required annotations for cert-manager, see\n ## ref: https://cert-manager.io/docs/usage/ingress/#supported-annotations\n ## e.g:\n ## annotations:\n ## kubernetes.io/ingress.class: nginx\n ## cert-manager.io/cluster-issuer: cluster-issuer-name\n ##\n annotations: {}\n ## @param ingress.tls Enable TLS configuration for the host defined at `ingress.hostname` parameter\n ## TLS certificates will be retrieved from a TLS secret with name: `{{- printf \"%s-tls\" .Values.ingress.hostname }}`\n ## You can:\n ## - Use the `ingress.secrets` parameter to create this TLS secret\n ## - Rely on cert-manager to create it by setting the corresponding annotations\n ## - Rely on Helm to create self-signed certificates by setting `ingress.selfSigned=true`\n ##\n tls: false\n ## @param ingress.selfSigned Create a TLS secret for this ingress record using self-signed certificates generated by Helm\n ##\n selfSigned: false\n ## @param ingress.extraHosts An array with additional hostname(s) to be covered with the ingress record\n ## e.g:\n ## extraHosts:\n ## - name: airflow.local\n ## path: /\n ##\n extraHosts: []\n ## @param ingress.extraPaths An array with additional arbitrary paths that may need to be added to the ingress under the main host\n ## e.g:\n ## extraPaths:\n ## - path: /*\n ## backend:\n ## serviceName: ssl-redirect\n ## servicePort: use-annotation\n ##\n extraPaths: []\n ## @param ingress.extraTls TLS configuration for additional hostname(s) to be covered with this ingress record\n ## ref: https://kubernetes.io/docs/concepts/services-networking/ingress/#tls\n ## e.g:\n ## extraTls:\n ## - hosts:\n ## - airflow.local\n ## secretName: airflow.local-tls\n ##\n extraTls: []\n ## @param ingress.secrets Custom TLS certificates as secrets\n ## NOTE: 'key' and 'certificate' are expected in PEM format\n ## NOTE: 'name' should line up with a 'secretName' set further up\n ## If it is not set and you're using cert-manager, this is unneeded, as it will create a secret for you with valid certificates\n ## If it is not set and you're NOT using cert-manager either, self-signed certificates will be created valid for 365 days\n ## It is also possible to create and manage the certificates outside of this helm chart\n ## Please see README.md for more information\n ## e.g:\n ## secrets:\n ## - name: airflow.local-tls\n ## key: |-\n ## -----BEGIN RSA PRIVATE KEY-----\n ## ...\n ## -----END RSA PRIVATE KEY-----\n ## certificate: |-\n ## -----BEGIN CERTIFICATE-----\n ## ...\n ## -----END CERTIFICATE-----\n ##\n secrets: []\n ## @param ingress.extraRules Additional rules to be covered with this ingress record\n ## ref: https://kubernetes.io/docs/concepts/services-networking/ingress/#ingress-rules\n ## e.g:\n ## extraRules:\n ## - host: example.local\n ## http:\n ## path: /\n ## backend:\n ## service:\n ## name: example-svc\n ## port:\n ## name: http\n ##\n extraRules: []\n\n## @section Other Parameters\n\n## Service account for Airflow pods to use.\n## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/\n##\nserviceAccount:\n ## @param serviceAccount.create Enable creation of ServiceAccount for Airflow pods\n ##\n create: false\n ## @param serviceAccount.name The name of the ServiceAccount to use.\n ## If not set and create is true, a name is generated using the common.names.fullname template\n ##\n name: \"\"\n ## @param serviceAccount.automountServiceAccountToken Allows auto mount of ServiceAccountToken on the serviceAccount created\n ## Can be set to false if pods using this serviceAccount do not need to use K8s API\n ##\n automountServiceAccountToken: true\n ## @param serviceAccount.annotations Additional custom annotations for the ServiceAccount\n ##\n annotations: {}\n## Role Based Access\n## Ref: https://kubernetes.io/docs/admin/authorization/rbac/\n## @param rbac.create Create Role and RoleBinding\n##\nrbac:\n create: false\n ## @param rbac.rules Custom RBAC rules to set\n ## e.g:\n ## rules:\n ## - apiGroups:\n ## - \"\"\n ## resources:\n ## - pods\n ## verbs:\n ## - get\n ## - list\n ##\n rules: []\n\n## @section Airflow metrics parameters\n\nmetrics:\n ## @param metrics.enabled Whether or not to create a standalone Airflow exporter to expose Airflow metrics\n ##\n enabled: false\n ## Bitnami Airflow exporter image\n ## ref: https://hub.docker.com/r/bitnami/airflow-exporter/tags/\n ## @param metrics.image.registry Airflow exporter image registry\n ## @param metrics.image.repository Airflow exporter image repository\n ## @param metrics.image.tag Airflow exporter image tag (immutable tags are recommended)\n ## @param metrics.image.digest Airflow exporter image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag\n ## @param metrics.image.pullPolicy Airflow exporter image pull policy\n ## @param metrics.image.pullSecrets Airflow exporter image pull secrets\n ##\n image:\n registry: docker.io\n repository: bitnami/airflow-exporter\n tag: 0.20220314.0-debian-11-r57\n digest: \"\"\n pullPolicy: IfNotPresent\n ## Optionally specify an array of imagePullSecrets.\n ## Secrets must be manually created in the namespace.\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/\n ## e.g:\n ## pullSecrets:\n ## - myRegistryKeySecretName\n ##\n pullSecrets: []\n ## @param metrics.extraEnvVars Array with extra environment variables to add Airflow exporter pods\n ##\n extraEnvVars: []\n ## @param metrics.extraEnvVarsCM ConfigMap containing extra environment variables for Airflow exporter pods\n ##\n extraEnvVarsCM: \"\"\n ## @param metrics.extraEnvVarsSecret Secret containing extra environment variables (in case of sensitive data) for Airflow exporter pods\n ##\n extraEnvVarsSecret: \"\"\n ## @param metrics.containerPorts.http Airflow exporter metrics container port\n ##\n containerPorts:\n http: 9112\n ## Airflow exporter resource requests and limits\n ## ref: https://kubernetes.io/docs/user-guide/compute-resources/\n ## @param metrics.resources.limits The resources limits for the container\n ## @param metrics.resources.requests The requested resources for the container\n ##\n resources:\n limits: {}\n requests: {}\n ## Airflow exporter pods' Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod\n ## @param metrics.podSecurityContext.enabled Enable security context for the pods\n ## @param metrics.podSecurityContext.fsGroup Set Airflow exporter pod's Security Context fsGroup\n ##\n podSecurityContext:\n enabled: true\n fsGroup: 1001\n ## Airflow exporter containers' Security Context\n ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container\n ## @param metrics.containerSecurityContext.enabled Enable Airflow exporter containers' Security Context\n ## @param metrics.containerSecurityContext.runAsUser Set Airflow exporter containers' Security Context runAsUser\n ## @param metrics.containerSecurityContext.runAsNonRoot Set Airflow exporter containers' Security Context runAsNonRoot\n ## e.g:\n ## containerSecurityContext:\n ## enabled: true\n ## capabilities:\n ## drop: [\"NET_RAW\"]\n ## readOnlyRootFilesystem: true\n ##\n containerSecurityContext:\n enabled: true\n runAsUser: 1001\n runAsNonRoot: true\n ## @param metrics.lifecycleHooks for the Airflow exporter container(s) to automate configuration before or after startup\n ##\n lifecycleHooks: {}\n ## @param metrics.hostAliases Airflow exporter pods host aliases\n ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/\n ##\n hostAliases: []\n ## @param metrics.podLabels Extra labels for Airflow exporter pods\n ## Ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/\n ##\n podLabels: {}\n ## @param metrics.podAnnotations Extra annotations for Airflow exporter pods\n ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/\n ##\n podAnnotations: {}\n ## @param metrics.podAffinityPreset Pod affinity preset. Ignored if `metrics.affinity` is set. Allowed values: `soft` or `hard`\n ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAffinityPreset: \"\"\n ## @param metrics.podAntiAffinityPreset Pod anti-affinity preset. Ignored if `metrics.affinity` is set. Allowed values: `soft` or `hard`\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity\n ##\n podAntiAffinityPreset: soft\n ## Node metrics.affinity preset\n ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity\n ##\n nodeAffinityPreset:\n ## @param metrics.nodeAffinityPreset.type Node affinity preset type. Ignored if `metrics.affinity` is set. Allowed values: `soft` or `hard`\n ##\n type: \"\"\n ## @param metrics.nodeAffinityPreset.key Node label key to match Ignored if `metrics.affinity` is set.\n ## E.g.\n ## key: \"kubernetes.io/e2e-az-name\"\n ##\n key: \"\"\n ## @param metrics.nodeAffinityPreset.values Node label values to match. Ignored if `metrics.affinity` is set.\n ## E.g.\n ## values:\n ## - e2e-az1\n ## - e2e-az2\n ##\n values: []\n ## @param metrics.affinity Affinity for pod assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity\n ## Note: metrics.podAffinityPreset, metrics.podAntiAffinityPreset, and metrics.nodeAffinityPreset will be ignored when it's set\n ##\n affinity: {}\n ## @param metrics.nodeSelector Node labels for pod assignment\n ## Ref: https://kubernetes.io/docs/user-guide/node-selection/\n ##\n nodeSelector: {}\n ## @param metrics.tolerations Tolerations for pod assignment\n ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/\n ##\n tolerations: []\n ## @param metrics.schedulerName Name of the k8s scheduler (other than default) for Airflow exporter\n ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/\n ##\n schedulerName: \"\"\n ## Airflow exporter service configuration\n ##\n service:\n ## @param metrics.service.ports.http Airflow exporter metrics service port\n ##\n ports:\n http: 9112\n ## @param metrics.service.clusterIP Static clusterIP or None for headless services\n ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#choosing-your-own-ip-address\n ##\n clusterIP: \"\"\n ## @param metrics.service.sessionAffinity Control where client requests go, to the same pod or round-robin\n ## Values: ClientIP or None\n ## ref: https://kubernetes.io/docs/user-guide/services/\n ##\n sessionAffinity: None\n ## @param metrics.service.annotations [object] Annotations for the Airflow exporter service\n ##\n annotations:\n prometheus.io/scrape: \"true\"\n prometheus.io/port: \"{{ .Values.metrics.service.ports.http }}\"\n ## Prometheus Operator ServiceMonitor configuration\n ##\n serviceMonitor:\n ## @param metrics.serviceMonitor.enabled if `true`, creates a Prometheus Operator ServiceMonitor (requires `metrics.enabled` to be `true`)\n ##\n enabled: false\n ## @param metrics.serviceMonitor.namespace Namespace in which Prometheus is running\n ##\n namespace: \"\"\n ## @param metrics.serviceMonitor.interval Interval at which metrics should be scraped\n ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint\n ##\n interval: \"\"\n ## @param metrics.serviceMonitor.scrapeTimeout Timeout after which the scrape is ended\n ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint\n ##\n scrapeTimeout: \"\"\n ## @param metrics.serviceMonitor.labels Additional labels that can be used so ServiceMonitor will be discovered by Prometheus\n ##\n labels: {}\n ## @param metrics.serviceMonitor.selector Prometheus instance selector labels\n ## ref: https://github.com/bitnami/charts/tree/main/bitnami/prometheus-operator#prometheus-configuration\n ##\n selector: {}\n ## @param metrics.serviceMonitor.relabelings RelabelConfigs to apply to samples before scraping\n ##\n relabelings: []\n ## @param metrics.serviceMonitor.metricRelabelings MetricRelabelConfigs to apply to samples before ingestion\n ##\n metricRelabelings: []\n ## @param metrics.serviceMonitor.honorLabels Specify honorLabels parameter to add the scrape endpoint\n ##\n honorLabels: false\n ## @param metrics.serviceMonitor.jobLabel The name of the label on the target service to use as the job name in prometheus.\n ##\n jobLabel: \"\"\n\n## @section Airflow database parameters\n\n## PostgreSQL chart configuration\n## ref: https://github.com/bitnami/charts/blob/main/bitnami/postgresql/values.yaml\n## @param postgresql.enabled Switch to enable or disable the PostgreSQL helm chart\n## @param postgresql.auth.enablePostgresUser Assign a password to the \"postgres\" admin user. Otherwise, remote access will be blocked for this user\n## @param postgresql.auth.username Name for a custom user to create\n## @param postgresql.auth.password Password for the custom user to create\n## @param postgresql.auth.database Name for a custom database to create\n## @param postgresql.auth.existingSecret Name of existing secret to use for PostgreSQL credentials\n## @param postgresql.architecture PostgreSQL architecture (`standalone` or `replication`)\n##\npostgresql:\n enabled: true\n auth:\n enablePostgresUser: false\n username: bn_airflow\n password: \"\"\n database: bitnami_airflow\n existingSecret: \"\"\n architecture: standalone\n## External PostgreSQL configuration\n## All of these values are only used when postgresql.enabled is set to false\n## @param externalDatabase.host Database host\n## @param externalDatabase.port Database port number\n## @param externalDatabase.user Non-root username for Airflow\n## @param externalDatabase.password Password for the non-root username for Airflow\n## @param externalDatabase.database Airflow database name\n## @param externalDatabase.existingSecret Name of an existing secret resource containing the database credentials\n## @param externalDatabase.existingSecretPasswordKey Name of an existing secret key containing the database credentials\n##\nexternalDatabase:\n host: localhost\n port: 5432\n user: bn_airflow\n database: bitnami_airflow\n password: \"\"\n existingSecret: \"\"\n existingSecretPasswordKey: \"\"\n\n## Redis® chart configuration\n## ref: https://github.com/bitnami/charts/blob/main/bitnami/redis/values.yaml\n## @param redis.enabled Switch to enable or disable the Redis® helm\n## @param redis.auth.enabled Enable password authentication\n## @param redis.auth.password Redis® password\n## @param redis.auth.existingSecret The name of an existing secret with Redis® credentials\n## @param redis.architecture Redis® architecture. Allowed values: `standalone` or `replication`\n##\nredis:\n enabled: true\n auth:\n enabled: true\n ## Redis® password (both master and slave). Defaults to a random 10-character alphanumeric string if not set and auth.enabled is true.\n ## It should always be set using the password value or in the existingSecret to avoid issues\n ## with Airflow.\n ## The password value is ignored if existingSecret is set\n password: \"\"\n existingSecret: \"\"\n architecture: standalone\n\n## External Redis® configuration\n## All of these values are only used when redis.enabled is set to false\n## @param externalRedis.host Redis® host\n## @param externalRedis.port Redis® port number\n## @param externalRedis.username Redis® username\n## @param externalRedis.password Redis® password\n## @param externalRedis.existingSecret Name of an existing secret resource containing the Redis&trade credentials\n## @param externalRedis.existingSecretPasswordKey Name of an existing secret key containing the Redis&trade credentials\n##\nexternalRedis:\n host: localhost\n port: 6379\n ## Most Redis® implementations do not require a username\n ## to authenticate and it should be enough with the password\n username: \"\"\n password: \"\"\n existingSecret: \"\"\n existingSecretPasswordKey: \"\"\n", Readme: "", UserId: 6, ReferenceValueId: 6304, diff --git a/pkg/deployment/gitOps/git/GitCliUtil.go b/pkg/deployment/gitOps/git/GitCliUtil.go index 587e20f1c6..1b54874f94 100644 --- a/pkg/deployment/gitOps/git/GitCliUtil.go +++ b/pkg/deployment/gitOps/git/GitCliUtil.go @@ -137,7 +137,7 @@ func (impl *GitCliUtil) Clone(rootDir string, remoteUrl string, username string, } response, errMsg, err = impl.Fetch(rootDir, username, password) if err == nil && errMsg == "" { - impl.logger.Warn("git fetch completed, pulling master branch data from git origin") + impl.logger.Warn("git fetch completed, pulling master branch data from remote origin") response, errMsg, err = impl.ListBranch(rootDir, username, password) if err != nil { impl.logger.Errorw("error on git pull", "response", response, "errMsg", errMsg, "err", err) diff --git a/pkg/pipeline/DeploymentPipelineConfigService.go b/pkg/pipeline/DeploymentPipelineConfigService.go index c806a1942e..5c7338ad5a 100644 --- a/pkg/pipeline/DeploymentPipelineConfigService.go +++ b/pkg/pipeline/DeploymentPipelineConfigService.go @@ -1593,7 +1593,7 @@ func (impl *CdPipelineConfigServiceImpl) RegisterInACD(gitOpsRepoName string, ch err := impl.argoClientWrapperService.RegisterGitOpsRepoInArgo(ctx, chartGitAttr.RepoUrl) if err != nil { impl.logger.Errorw("error while register git repo in argo", "err", err) - emptyRepoErrorMessage := []string{"failed to get index: 404 Not Found", "git repository is empty"} + emptyRepoErrorMessage := []string{"failed to get index: 404 Not Found", "remote repository is empty"} if strings.Contains(err.Error(), emptyRepoErrorMessage[0]) || strings.Contains(err.Error(), emptyRepoErrorMessage[1]) { // - found empty repository, create some file in repository err := impl.gitOperationService.CreateReadmeInGitRepo(gitOpsRepoName, userId) diff --git a/pkg/pipeline/PipelineStageServiceIT_test.go b/pkg/pipeline/PipelineStageServiceIT_test.go index 532d509eb7..f6e8d803c5 100644 --- a/pkg/pipeline/PipelineStageServiceIT_test.go +++ b/pkg/pipeline/PipelineStageServiceIT_test.go @@ -26,9 +26,9 @@ import ( "time" ) -const CiPipelineStageCreateReqJson = `{"appId":1,"appWorkflowId":0,"action":0,"ciPipeline":{"active":true,"ciMaterial":[{"gitMaterialId":1,"id":0,"source":{"type":"SOURCE_TYPE_BRANCH_FIXED","value":"main","regex":""}}],"dockerArgs":{},"externalCiConfig":{},"id":0,"isExternal":false,"isManual":false,"name":"ci-1-xyze","linkedCount":0,"scanEnabled":false,"preBuildStage":{"id":0,"steps":[{"id":1,"index":1,"name":"Task 1","description":"chbsdkhbc","stepType":"INLINE","directoryPath":"","inlineStepDetail":{"scriptType":"CONTAINER_IMAGE","script":"echo \"ifudsbvnv\"","conditionDetails":[],"inputVariables":[{"id":1,"name":"Hello","value":"","format":"STRING","description":"jnsdvbdvbsd","defaultValue":"","variableType":"GLOBAL","refVariableStepIndex":0,"refVariableName":"WORKING_DIRECTORY","refVariableStage":""}],"outputVariables":null,"commandArgsMap":[{"command":"echo","args":["\"HOSTNAME\"","\"PORT\""]}],"portMap":[{"portOnLocal":8080,"portOnContainer":9090}],"mountCodeToContainer":true,"mountDirectoryFromHost":true,"mountCodeToContainerPath":"/sourcecode","mountPathMap":[{"filePathOnDisk":"./test","filePathOnContainer":"./test_container"}],"containerImagePath":"python:latest","isMountCustomScript":true,"storeScriptAt":"./directory/script"},"outputDirectoryPath":["./test1"]},{"id":2,"index":2,"name":"K6 Load testing","description":"K6 is an open-source tool and cloud service that makes load testing easy for developers and QA engineers.","stepType":"REF_PLUGIN","directoryPath":"","pluginRefStepDetail":{"id":0,"pluginId":1,"conditionDetails":[{"id":0,"conditionOnVariable":"RelativePathToScript","conditionOperator":"==","conditionType":"TRIGGER","conditionalValue":"svfsv"},{"id":0,"conditionOnVariable":"PrometheusApiKey","conditionOperator":"==","conditionType":"TRIGGER","conditionalValue":"dfbeavafsv"}],"inputVariables":[{"id":1,"name":"RelativePathToScript","format":"STRING","description":"checkout path + script path along with script name","isExposed":true,"allowEmptyValue":false,"defaultValue":"/./script.js","variableType":"NEW","variableStepIndexInPlugin":1,"value":"dethdt","refVariableName":"","refVariableStage":""},{"id":2,"name":"PrometheusUsername","format":"STRING","description":"username of prometheus account","isExposed":true,"allowEmptyValue":true,"defaultValue":"","variableType":"NEW","variableStepIndexInPlugin":1,"value":"ghmfnbd","refVariableName":"","refVariableStage":""},{"id":3,"name":"PrometheusApiKey","format":"STRING","description":"api key of prometheus account","isExposed":true,"allowEmptyValue":true,"defaultValue":"","variableType":"NEW","variableStepIndexInPlugin":1,"value":"afegs","refVariableName":"","refVariableStage":""},{"id":4,"name":"PrometheusRemoteWriteEndpoint","format":"STRING","description":"git write endpoint of prometheus account","isExposed":true,"allowEmptyValue":true,"defaultValue":"","variableType":"NEW","variableStepIndexInPlugin":1,"value":"aef","refVariableName":"","refVariableStage":""},{"id":5,"name":"OutputType","format":"STRING","description":"output type - LOG or PROMETHEUS","isExposed":true,"allowEmptyValue":false,"defaultValue":"LOG","variableType":"NEW","variableStepIndexInPlugin":1,"value":"fdgn","refVariableName":"","refVariableStage":""}]}},{"id":3,"index":3,"name":"Task 3","description":"sfdbvf","stepType":"INLINE","directoryPath":"","inlineStepDetail":{"scriptType":"SHELL","script":"#!/bin/sh \nset -eo pipefail \n#set -v ## uncomment this to debug the script \n","conditionDetails":[{"id":0,"conditionOnVariable":"Hello","conditionOperator":"==","conditionType":"PASS","conditionalValue":"aedfrwgwr"},{"id":0,"conditionOnVariable":"Hello","conditionOperator":"!=","conditionType":"PASS","conditionalValue":"tegegr"}],"inputVariables":[],"outputVariables":[{"id":1,"name":"Hello","value":"","format":"STRING","description":"dsuihvsuvhbdv","defaultValue":"","variableType":"NEW","refVariableStepIndex":0,"refVariableName":""}],"commandArgsMap":[{"command":"","args":[]}],"portMap":[],"mountCodeToContainer":false,"mountDirectoryFromHost":false},"outputDirectoryPath":["./test2"]}]},"postBuildStage":{},"dockerConfigOverride":{}}}` -const CiPipelineStageUpdateReqJson = `{"appId":1,"appWorkflowId":3,"action":1,"ciPipeline":{"isManual":false,"dockerArgs":{},"isExternal":false,"parentCiPipeline":0,"parentAppId":0,"appId":1,"externalCiConfig":{"id":0,"webhookUrl":"","payload":"","accessKey":"","payloadOption":null,"schema":null,"responses":null,"projectId":0,"projectName":"","environmentId":"","environmentName":"","environmentIdentifier":"","appId":0,"appName":"","role":""},"ciMaterial":[{"gitMaterialId":1,"id":3,"source":{"type":"SOURCE_TYPE_BRANCH_FIXED","value":"main","regex":""}}],"name":"ci-1-unov","id":3,"active":true,"linkedCount":0,"scanEnabled":false,"appWorkflowId":3,"preBuildStage":{"id":5,"type":"PRE_CI","steps":[{"id":9,"name":"K6 Load testing","description":"K6 is an open-source tool and cloud service that makes load testing easy for developers and QA engineers.","index":1,"stepType":"REF_PLUGIN","outputDirectoryPath":null,"inlineStepDetail":null,"pluginRefStepDetail":{"pluginId":1,"inputVariables":[{"id":44,"name":"RelativePathToScript","format":"STRING","description":"checkout path + script path along with script name","isExposed":true,"defaultValue":"/./script.js","value":"sfds","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""},{"id":45,"name":"PrometheusUsername","format":"STRING","description":"username of prometheus account","isExposed":true,"allowEmptyValue":true,"value":"sdf","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""},{"id":46,"name":"PrometheusApiKey","format":"STRING","description":"api key of prometheus account","isExposed":true,"allowEmptyValue":true,"value":"hter","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""},{"id":47,"name":"PrometheusRemoteWriteEndpoint","format":"STRING","description":"git write endpoint of prometheus account","isExposed":true,"allowEmptyValue":true,"value":"ewq","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""},{"id":48,"name":"OutputType","format":"STRING","description":"output type - LOG or PROMETHEUS","isExposed":true,"defaultValue":"LOG","value":"erg","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""}],"outputVariables":null,"conditionDetails":null},"triggerIfParentStageFail":false},{"id":10,"name":"Task 3","description":"","index":3,"stepType":"INLINE","outputDirectoryPath":["./asap"],"inlineStepDetail":{"scriptType":"CONTAINER_IMAGE","script":null,"storeScriptAt":null,"mountDirectoryFromHost":false,"containerImagePath":"alpine:latest","commandArgsMap":[{"command":"echo","args":["HOSTNAME"]}],"inputVariables":null,"outputVariables":null,"conditionDetails":null,"portMap":[],"mountCodeToContainerPath":null,"mountPathMap":null},"pluginRefStepDetail":null,"triggerIfParentStageFail":false},{"id":8,"name":"Task 1","description":"","index":2,"stepType":"INLINE","outputDirectoryPath":["./test"],"inlineStepDetail":{"scriptType":"SHELL","script":"#!/bin/sh \nset -eo pipefail \necho \"Hello from inside pre-build stage\"\n#set -v ## uncomment this to debug the script \n","storeScriptAt":"","mountDirectoryFromHost":false,"commandArgsMap":[{"command":"","args":null}],"inputVariables":null,"outputVariables":null,"conditionDetails":null},"pluginRefStepDetail":null,"triggerIfParentStageFail":false}]},"postBuildStage":{},"isDockerConfigOverridden":false,"dockerConfigOverride":{}}}` -const CiPipelineStageDeleteReqJson = `{"appId":1,"appWorkflowId":8,"action":2,"ciPipeline":{"isManual":false,"dockerArgs":{},"isExternal":false,"parentCiPipeline":0,"parentAppId":0,"appId":1,"externalCiConfig":{"id":0,"webhookUrl":"","payload":"","accessKey":"","payloadOption":null,"schema":null,"responses":null,"projectId":0,"projectName":"","environmentId":"","environmentName":"","environmentIdentifier":"","appId":0,"appName":"","role":""},"ciMaterial":[{"gitMaterialId":1,"id":8,"source":{"type":"SOURCE_TYPE_BRANCH_FIXED","value":"main","regex":""}}],"name":"ci-1-unjn","id":8,"active":true,"linkedCount":0,"scanEnabled":false,"appWorkflowId":8,"preBuildStage":{"id":7,"type":"PRE_CI","steps":[{"id":11,"name":"Task 1","description":"","index":1,"stepType":"INLINE","outputDirectoryPath":["./test"],"inlineStepDetail":{"scriptType":"SHELL","script":"#!/bin/sh \nset -eo pipefail \necho \"Prakash\"\n#set -v ## uncomment this to debug the script \n","storeScriptAt":"","mountDirectoryFromHost":false,"commandArgsMap":[{"command":"","args":null}],"inputVariables":null,"outputVariables":null,"conditionDetails":null},"pluginRefStepDetail":null,"triggerIfParentStageFail":false},{"id":12,"name":"K6 Load testing","description":"K6 is an open-source tool and cloud service that makes load testing easy for developers and QA engineers.","index":2,"stepType":"REF_PLUGIN","outputDirectoryPath":null,"inlineStepDetail":null,"pluginRefStepDetail":{"pluginId":1,"inputVariables":[{"id":49,"name":"RelativePathToScript","format":"STRING","description":"checkout path + script path along with script name","isExposed":true,"defaultValue":"/./script.js","value":"sfds","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""},{"id":50,"name":"PrometheusUsername","format":"STRING","description":"username of prometheus account","isExposed":true,"allowEmptyValue":true,"value":"sdf","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""},{"id":51,"name":"PrometheusApiKey","format":"STRING","description":"api key of prometheus account","isExposed":true,"allowEmptyValue":true,"value":"hter","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""},{"id":52,"name":"PrometheusRemoteWriteEndpoint","format":"STRING","description":"git write endpoint of prometheus account","isExposed":true,"allowEmptyValue":true,"value":"ewq","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""},{"id":53,"name":"OutputType","format":"STRING","description":"output type - LOG or PROMETHEUS","isExposed":true,"defaultValue":"LOG","value":"erg","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""}],"outputVariables":null,"conditionDetails":null},"triggerIfParentStageFail":false},{"id":13,"name":"Task 3","description":"","index":3,"stepType":"INLINE","outputDirectoryPath":["./asap"],"inlineStepDetail":{"scriptType":"CONTAINER_IMAGE","script":"","storeScriptAt":"","mountDirectoryFromHost":false,"containerImagePath":"alpine:latest","commandArgsMap":[{"command":"echo","args":["HOSTNAME"]}],"inputVariables":null,"outputVariables":null,"conditionDetails":null,"portMap":[]},"pluginRefStepDetail":null,"triggerIfParentStageFail":false}]},"isDockerConfigOverridden":false,"dockerConfigOverride":{},"postBuildStage":{}}}` +const CiPipelineStageCreateReqJson = `{"appId":1,"appWorkflowId":0,"action":0,"ciPipeline":{"active":true,"ciMaterial":[{"gitMaterialId":1,"id":0,"source":{"type":"SOURCE_TYPE_BRANCH_FIXED","value":"main","regex":""}}],"dockerArgs":{},"externalCiConfig":{},"id":0,"isExternal":false,"isManual":false,"name":"ci-1-xyze","linkedCount":0,"scanEnabled":false,"preBuildStage":{"id":0,"steps":[{"id":1,"index":1,"name":"Task 1","description":"chbsdkhbc","stepType":"INLINE","directoryPath":"","inlineStepDetail":{"scriptType":"CONTAINER_IMAGE","script":"echo \"ifudsbvnv\"","conditionDetails":[],"inputVariables":[{"id":1,"name":"Hello","value":"","format":"STRING","description":"jnsdvbdvbsd","defaultValue":"","variableType":"GLOBAL","refVariableStepIndex":0,"refVariableName":"WORKING_DIRECTORY","refVariableStage":""}],"outputVariables":null,"commandArgsMap":[{"command":"echo","args":["\"HOSTNAME\"","\"PORT\""]}],"portMap":[{"portOnLocal":8080,"portOnContainer":9090}],"mountCodeToContainer":true,"mountDirectoryFromHost":true,"mountCodeToContainerPath":"/sourcecode","mountPathMap":[{"filePathOnDisk":"./test","filePathOnContainer":"./test_container"}],"containerImagePath":"python:latest","isMountCustomScript":true,"storeScriptAt":"./directory/script"},"outputDirectoryPath":["./test1"]},{"id":2,"index":2,"name":"K6 Load testing","description":"K6 is an open-source tool and cloud service that makes load testing easy for developers and QA engineers.","stepType":"REF_PLUGIN","directoryPath":"","pluginRefStepDetail":{"id":0,"pluginId":1,"conditionDetails":[{"id":0,"conditionOnVariable":"RelativePathToScript","conditionOperator":"==","conditionType":"TRIGGER","conditionalValue":"svfsv"},{"id":0,"conditionOnVariable":"PrometheusApiKey","conditionOperator":"==","conditionType":"TRIGGER","conditionalValue":"dfbeavafsv"}],"inputVariables":[{"id":1,"name":"RelativePathToScript","format":"STRING","description":"checkout path + script path along with script name","isExposed":true,"allowEmptyValue":false,"defaultValue":"/./script.js","variableType":"NEW","variableStepIndexInPlugin":1,"value":"dethdt","refVariableName":"","refVariableStage":""},{"id":2,"name":"PrometheusUsername","format":"STRING","description":"username of prometheus account","isExposed":true,"allowEmptyValue":true,"defaultValue":"","variableType":"NEW","variableStepIndexInPlugin":1,"value":"ghmfnbd","refVariableName":"","refVariableStage":""},{"id":3,"name":"PrometheusApiKey","format":"STRING","description":"api key of prometheus account","isExposed":true,"allowEmptyValue":true,"defaultValue":"","variableType":"NEW","variableStepIndexInPlugin":1,"value":"afegs","refVariableName":"","refVariableStage":""},{"id":4,"name":"PrometheusRemoteWriteEndpoint","format":"STRING","description":"remote write endpoint of prometheus account","isExposed":true,"allowEmptyValue":true,"defaultValue":"","variableType":"NEW","variableStepIndexInPlugin":1,"value":"aef","refVariableName":"","refVariableStage":""},{"id":5,"name":"OutputType","format":"STRING","description":"output type - LOG or PROMETHEUS","isExposed":true,"allowEmptyValue":false,"defaultValue":"LOG","variableType":"NEW","variableStepIndexInPlugin":1,"value":"fdgn","refVariableName":"","refVariableStage":""}]}},{"id":3,"index":3,"name":"Task 3","description":"sfdbvf","stepType":"INLINE","directoryPath":"","inlineStepDetail":{"scriptType":"SHELL","script":"#!/bin/sh \nset -eo pipefail \n#set -v ## uncomment this to debug the script \n","conditionDetails":[{"id":0,"conditionOnVariable":"Hello","conditionOperator":"==","conditionType":"PASS","conditionalValue":"aedfrwgwr"},{"id":0,"conditionOnVariable":"Hello","conditionOperator":"!=","conditionType":"PASS","conditionalValue":"tegegr"}],"inputVariables":[],"outputVariables":[{"id":1,"name":"Hello","value":"","format":"STRING","description":"dsuihvsuvhbdv","defaultValue":"","variableType":"NEW","refVariableStepIndex":0,"refVariableName":""}],"commandArgsMap":[{"command":"","args":[]}],"portMap":[],"mountCodeToContainer":false,"mountDirectoryFromHost":false},"outputDirectoryPath":["./test2"]}]},"postBuildStage":{},"dockerConfigOverride":{}}}` +const CiPipelineStageUpdateReqJson = `{"appId":1,"appWorkflowId":3,"action":1,"ciPipeline":{"isManual":false,"dockerArgs":{},"isExternal":false,"parentCiPipeline":0,"parentAppId":0,"appId":1,"externalCiConfig":{"id":0,"webhookUrl":"","payload":"","accessKey":"","payloadOption":null,"schema":null,"responses":null,"projectId":0,"projectName":"","environmentId":"","environmentName":"","environmentIdentifier":"","appId":0,"appName":"","role":""},"ciMaterial":[{"gitMaterialId":1,"id":3,"source":{"type":"SOURCE_TYPE_BRANCH_FIXED","value":"main","regex":""}}],"name":"ci-1-unov","id":3,"active":true,"linkedCount":0,"scanEnabled":false,"appWorkflowId":3,"preBuildStage":{"id":5,"type":"PRE_CI","steps":[{"id":9,"name":"K6 Load testing","description":"K6 is an open-source tool and cloud service that makes load testing easy for developers and QA engineers.","index":1,"stepType":"REF_PLUGIN","outputDirectoryPath":null,"inlineStepDetail":null,"pluginRefStepDetail":{"pluginId":1,"inputVariables":[{"id":44,"name":"RelativePathToScript","format":"STRING","description":"checkout path + script path along with script name","isExposed":true,"defaultValue":"/./script.js","value":"sfds","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""},{"id":45,"name":"PrometheusUsername","format":"STRING","description":"username of prometheus account","isExposed":true,"allowEmptyValue":true,"value":"sdf","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""},{"id":46,"name":"PrometheusApiKey","format":"STRING","description":"api key of prometheus account","isExposed":true,"allowEmptyValue":true,"value":"hter","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""},{"id":47,"name":"PrometheusRemoteWriteEndpoint","format":"STRING","description":"remote write endpoint of prometheus account","isExposed":true,"allowEmptyValue":true,"value":"ewq","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""},{"id":48,"name":"OutputType","format":"STRING","description":"output type - LOG or PROMETHEUS","isExposed":true,"defaultValue":"LOG","value":"erg","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""}],"outputVariables":null,"conditionDetails":null},"triggerIfParentStageFail":false},{"id":10,"name":"Task 3","description":"","index":3,"stepType":"INLINE","outputDirectoryPath":["./asap"],"inlineStepDetail":{"scriptType":"CONTAINER_IMAGE","script":null,"storeScriptAt":null,"mountDirectoryFromHost":false,"containerImagePath":"alpine:latest","commandArgsMap":[{"command":"echo","args":["HOSTNAME"]}],"inputVariables":null,"outputVariables":null,"conditionDetails":null,"portMap":[],"mountCodeToContainerPath":null,"mountPathMap":null},"pluginRefStepDetail":null,"triggerIfParentStageFail":false},{"id":8,"name":"Task 1","description":"","index":2,"stepType":"INLINE","outputDirectoryPath":["./test"],"inlineStepDetail":{"scriptType":"SHELL","script":"#!/bin/sh \nset -eo pipefail \necho \"Hello from inside pre-build stage\"\n#set -v ## uncomment this to debug the script \n","storeScriptAt":"","mountDirectoryFromHost":false,"commandArgsMap":[{"command":"","args":null}],"inputVariables":null,"outputVariables":null,"conditionDetails":null},"pluginRefStepDetail":null,"triggerIfParentStageFail":false}]},"postBuildStage":{},"isDockerConfigOverridden":false,"dockerConfigOverride":{}}}` +const CiPipelineStageDeleteReqJson = `{"appId":1,"appWorkflowId":8,"action":2,"ciPipeline":{"isManual":false,"dockerArgs":{},"isExternal":false,"parentCiPipeline":0,"parentAppId":0,"appId":1,"externalCiConfig":{"id":0,"webhookUrl":"","payload":"","accessKey":"","payloadOption":null,"schema":null,"responses":null,"projectId":0,"projectName":"","environmentId":"","environmentName":"","environmentIdentifier":"","appId":0,"appName":"","role":""},"ciMaterial":[{"gitMaterialId":1,"id":8,"source":{"type":"SOURCE_TYPE_BRANCH_FIXED","value":"main","regex":""}}],"name":"ci-1-unjn","id":8,"active":true,"linkedCount":0,"scanEnabled":false,"appWorkflowId":8,"preBuildStage":{"id":7,"type":"PRE_CI","steps":[{"id":11,"name":"Task 1","description":"","index":1,"stepType":"INLINE","outputDirectoryPath":["./test"],"inlineStepDetail":{"scriptType":"SHELL","script":"#!/bin/sh \nset -eo pipefail \necho \"Prakash\"\n#set -v ## uncomment this to debug the script \n","storeScriptAt":"","mountDirectoryFromHost":false,"commandArgsMap":[{"command":"","args":null}],"inputVariables":null,"outputVariables":null,"conditionDetails":null},"pluginRefStepDetail":null,"triggerIfParentStageFail":false},{"id":12,"name":"K6 Load testing","description":"K6 is an open-source tool and cloud service that makes load testing easy for developers and QA engineers.","index":2,"stepType":"REF_PLUGIN","outputDirectoryPath":null,"inlineStepDetail":null,"pluginRefStepDetail":{"pluginId":1,"inputVariables":[{"id":49,"name":"RelativePathToScript","format":"STRING","description":"checkout path + script path along with script name","isExposed":true,"defaultValue":"/./script.js","value":"sfds","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""},{"id":50,"name":"PrometheusUsername","format":"STRING","description":"username of prometheus account","isExposed":true,"allowEmptyValue":true,"value":"sdf","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""},{"id":51,"name":"PrometheusApiKey","format":"STRING","description":"api key of prometheus account","isExposed":true,"allowEmptyValue":true,"value":"hter","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""},{"id":52,"name":"PrometheusRemoteWriteEndpoint","format":"STRING","description":"remote write endpoint of prometheus account","isExposed":true,"allowEmptyValue":true,"value":"ewq","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""},{"id":53,"name":"OutputType","format":"STRING","description":"output type - LOG or PROMETHEUS","isExposed":true,"defaultValue":"LOG","value":"erg","variableType":"NEW","variableStepIndexInPlugin":1,"refVariableStage":""}],"outputVariables":null,"conditionDetails":null},"triggerIfParentStageFail":false},{"id":13,"name":"Task 3","description":"","index":3,"stepType":"INLINE","outputDirectoryPath":["./asap"],"inlineStepDetail":{"scriptType":"CONTAINER_IMAGE","script":"","storeScriptAt":"","mountDirectoryFromHost":false,"containerImagePath":"alpine:latest","commandArgsMap":[{"command":"echo","args":["HOSTNAME"]}],"inputVariables":null,"outputVariables":null,"conditionDetails":null,"portMap":[]},"pluginRefStepDetail":null,"triggerIfParentStageFail":false}]},"isDockerConfigOverridden":false,"dockerConfigOverride":{},"postBuildStage":{}}}` const ScopedVariablePayload = `{"variables":[{"definition":{"varName":"KAFKA","dataType":"primitive","varType":"public","description":"Enter any notes for additional details","shortDescription":"Some description for variables"},"attributeValue":[{"variableValue":{"value":"Global"},"attributeType":"Global","attributeParams":null}]},{"definition":{"varName":"NewVariable","dataType":"primitive","varType":"private","description":"New Notes","shortDescription":"New Descriotion"},"attributeValue":[{"variableValue":{"value":"gg"},"attributeType":"Global","attributeParams":null}]},{"definition":{"varName":"test-var-1","dataType":"primitive","varType":"public","description":"","shortDescription":""},"attributeValue":[{"variableValue":{"value":"test1"},"attributeType":"Global","attributeParams":null}]},{"definition":{"varName":"test-var-2","dataType":"primitive","varType":"public","description":"","shortDescription":""},"attributeValue":[{"variableValue":{"value":"test2"},"attributeType":"Global","attributeParams":null}]}]}` var ciPipelineId int @@ -143,7 +143,7 @@ var pipelineStageReq = &bean.PipelineStageDto{ Id: 4, Name: "PrometheusRemoteWriteEndpoint", Format: "STRING", - Description: "git write endpoint of prometheus account", + Description: "remote write endpoint of prometheus account", IsExposed: true, AllowEmptyValue: true, DefaultValue: "", @@ -323,7 +323,7 @@ var pipelineStageReqUnresolved = &bean.PipelineStageDto{ Id: 4, Name: "PrometheusRemoteWriteEndpoint", Format: "STRING", - Description: "git write endpoint of prometheus account", + Description: "remote write endpoint of prometheus account", IsExposed: true, AllowEmptyValue: true, DefaultValue: "", @@ -504,7 +504,7 @@ var pipelineStageReqResolved = &bean.PipelineStageDto{ Id: 4, Name: "PrometheusRemoteWriteEndpoint", Format: "STRING", - Description: "git write endpoint of prometheus account", + Description: "remote write endpoint of prometheus account", IsExposed: true, AllowEmptyValue: true, DefaultValue: "", diff --git a/util/RequestUtil.go b/util/RequestUtil.go index eb0b251ec9..3eb349bc3d 100644 --- a/util/RequestUtil.go +++ b/util/RequestUtil.go @@ -24,7 +24,7 @@ import ( const xForwardedForHeaderName = "X-Forwarded-For" // GetClientIP gets a requests IP address by reading off the forwarded-for -// header (for proxies) and falls back to use the git address. +// header (for proxies) and falls back to use the remote address. func GetClientIP(r *http.Request) string { xForwardedFor := r.Header.Get(xForwardedForHeaderName) if len(xForwardedFor) > 0 { From 0af9223e4f41496124f688f764c875a89064d60e Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Thu, 25 Jan 2024 12:51:53 +0530 Subject: [PATCH 42/64] wip --- client/telemetry/TelemetryEventClient.go | 2 +- pkg/cluster/ClusterServiceExtended.go | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/client/telemetry/TelemetryEventClient.go b/client/telemetry/TelemetryEventClient.go index c806f42f7f..5b34cd95c9 100644 --- a/client/telemetry/TelemetryEventClient.go +++ b/client/telemetry/TelemetryEventClient.go @@ -67,7 +67,7 @@ type TelemetryEventClient interface { SendTelemetryDashboardLoggedInEvent() error SendGenericTelemetryEvent(eventType string, prop map[string]interface{}) error SendSummaryEvent(eventType string) error -} +}w func NewTelemetryEventClientImpl(logger *zap.SugaredLogger, client *http.Client, clusterService cluster.ClusterService, K8sUtil *k8s.K8sServiceImpl, aCDAuthConfig *util3.ACDAuthConfig, userService user2.UserService, diff --git a/pkg/cluster/ClusterServiceExtended.go b/pkg/cluster/ClusterServiceExtended.go index e18865e2b9..0982b9b9f4 100644 --- a/pkg/cluster/ClusterServiceExtended.go +++ b/pkg/cluster/ClusterServiceExtended.go @@ -43,10 +43,11 @@ func NewClusterServiceImplExtended(repository repository.ClusterRepository, envi userRepository repository5.UserRepository, roleGroupRepository repository5.RoleGroupRepository, gitOpsConfigReadService config.GitOpsConfigReadService) *ClusterServiceImplExtended { clusterServiceExt := &ClusterServiceImplExtended{ - environmentRepository: environmentRepository, - grafanaClient: grafanaClient, - installedAppRepository: installedAppRepository, - clusterServiceCD: clusterServiceCD, + environmentRepository: environmentRepository, + grafanaClient: grafanaClient, + installedAppRepository: installedAppRepository, + clusterServiceCD: clusterServiceCD, + gitOpsConfigReadService: gitOpsConfigReadService, ClusterServiceImpl: &ClusterServiceImpl{ clusterRepository: repository, logger: logger, From 799ffaec59461d2f01cb8c79d4e8f824120da5f6 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Thu, 25 Jan 2024 13:45:32 +0530 Subject: [PATCH 43/64] wip --- .../DeploymentTemplateRepository.go | 6 ++--- .../DeployementTemplateService.go | 4 ++-- .../DeployementTemplateService_test.go | 24 +++++++++---------- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/internal/sql/repository/DeploymentTemplateRepository.go b/internal/sql/repository/DeploymentTemplateRepository.go index c96760ece5..c35df61721 100644 --- a/internal/sql/repository/DeploymentTemplateRepository.go +++ b/internal/sql/repository/DeploymentTemplateRepository.go @@ -16,7 +16,7 @@ const ( ) type DeploymentTemplateComparisonMetadata struct { - ChartId int `json:"chartRefId"` + ChartRefId int `json:"chartRefId"` ChartVersion string `json:"chartVersion,omitempty"` ChartType string `json:"chartType,omitempty"` EnvironmentId int `json:"environmentId,omitempty"` @@ -78,7 +78,7 @@ func (impl DeploymentTemplateRepositoryImpl) FetchLatestDeploymentWithChartRefs( p.id as pipeline_id, p.environment_id, dth.id as deployment_template_history_id, - ceco.chart_id, + c.chart_ref_id, c.chart_version, ROW_NUMBER() OVER (PARTITION BY p.environment_id ORDER BY pco.id DESC) AS row_num FROM @@ -100,7 +100,7 @@ func (impl DeploymentTemplateRepositoryImpl) FetchLatestDeploymentWithChartRefs( rr.pipeline_id, rr.environment_id, rr.deployment_template_history_id, - rr.chart_id, + rr.chart_ref_id, rr.chart_version, e.environment_name FROM diff --git a/pkg/generateManifest/DeployementTemplateService.go b/pkg/generateManifest/DeployementTemplateService.go index cc5bf594dc..82b8a909a8 100644 --- a/pkg/generateManifest/DeployementTemplateService.go +++ b/pkg/generateManifest/DeployementTemplateService.go @@ -135,7 +135,7 @@ func (impl DeploymentTemplateServiceImpl) FetchDeploymentsWithChartRefs(appId in for _, item := range defaultVersions.ChartRefs { res := &repository.DeploymentTemplateComparisonMetadata{ - ChartId: item.Id, + ChartRefId: item.Id, ChartVersion: item.Version, ChartType: item.Name, Type: repository.DefaultVersions, @@ -151,7 +151,7 @@ func (impl DeploymentTemplateServiceImpl) FetchDeploymentsWithChartRefs(appId in for _, env := range publishedOnEnvs { item := &repository.DeploymentTemplateComparisonMetadata{ - ChartId: env.ChartRefId, + ChartRefId: env.ChartRefId, EnvironmentId: env.EnvironmentId, EnvironmentName: env.EnvironmentName, Type: repository.PublishedOnEnvironments, diff --git a/pkg/generateManifest/DeployementTemplateService_test.go b/pkg/generateManifest/DeployementTemplateService_test.go index 43bce354d7..d2aac53148 100644 --- a/pkg/generateManifest/DeployementTemplateService_test.go +++ b/pkg/generateManifest/DeployementTemplateService_test.go @@ -65,7 +65,7 @@ func TestDeploymentTemplateServiceImpl_FetchDeploymentsWithChartRefs(t *testing. deployedOnEnv := []*repository.DeploymentTemplateComparisonMetadata{ { - ChartId: 1, + ChartRefId: 1, ChartVersion: "4.18.1", EnvironmentId: 1, PipelineConfigOverrideId: 5, @@ -74,7 +74,7 @@ func TestDeploymentTemplateServiceImpl_FetchDeploymentsWithChartRefs(t *testing. Status: "Succeeded", }, { - ChartId: 1, + ChartRefId: 1, ChartVersion: "4.18.1", EnvironmentId: 1, PipelineConfigOverrideId: 5, @@ -83,7 +83,7 @@ func TestDeploymentTemplateServiceImpl_FetchDeploymentsWithChartRefs(t *testing. Status: "Succeeded", }, { - ChartId: 1, + ChartRefId: 1, ChartVersion: "4.18.1", EnvironmentId: 1, PipelineConfigOverrideId: 5, @@ -95,7 +95,7 @@ func TestDeploymentTemplateServiceImpl_FetchDeploymentsWithChartRefs(t *testing. deployedOnOtherEnvs := []*repository.DeploymentTemplateComparisonMetadata{ { - ChartId: 1, + ChartRefId: 1, ChartVersion: "4.18.1", EnvironmentId: 2, PipelineConfigOverrideId: 9, @@ -121,7 +121,7 @@ func TestDeploymentTemplateServiceImpl_FetchDeploymentsWithChartRefs(t *testing. }, want: []*repository.DeploymentTemplateComparisonMetadata{ { - ChartId: 1, + ChartRefId: 1, ChartVersion: "v1.0.1", ChartType: "Deployment", EnvironmentId: 0, @@ -133,7 +133,7 @@ func TestDeploymentTemplateServiceImpl_FetchDeploymentsWithChartRefs(t *testing. Type: 1, }, { - ChartId: 2, + ChartRefId: 2, ChartVersion: "v1.0.2", ChartType: "Deployment", EnvironmentId: 0, @@ -144,7 +144,7 @@ func TestDeploymentTemplateServiceImpl_FetchDeploymentsWithChartRefs(t *testing. Status: "", Type: 1, }, { - ChartId: 3, + ChartRefId: 3, ChartVersion: "v1.0.3", ChartType: "Deployment", EnvironmentId: 0, @@ -155,7 +155,7 @@ func TestDeploymentTemplateServiceImpl_FetchDeploymentsWithChartRefs(t *testing. Status: "", Type: 1, }, { - ChartId: 2, + ChartRefId: 2, ChartVersion: "", ChartType: "", EnvironmentId: 1, @@ -166,7 +166,7 @@ func TestDeploymentTemplateServiceImpl_FetchDeploymentsWithChartRefs(t *testing. Status: "", Type: 2, }, { - ChartId: 1, + ChartRefId: 1, ChartVersion: "4.18.1", ChartType: "", EnvironmentId: 1, @@ -177,7 +177,7 @@ func TestDeploymentTemplateServiceImpl_FetchDeploymentsWithChartRefs(t *testing. Status: "Succeeded", Type: 3, }, { - ChartId: 1, + ChartRefId: 1, ChartVersion: "4.18.1", ChartType: "", EnvironmentId: 1, @@ -188,7 +188,7 @@ func TestDeploymentTemplateServiceImpl_FetchDeploymentsWithChartRefs(t *testing. Status: "Succeeded", Type: 3, }, { - ChartId: 1, + ChartRefId: 1, ChartVersion: "4.18.1", ChartType: "", EnvironmentId: 1, @@ -199,7 +199,7 @@ func TestDeploymentTemplateServiceImpl_FetchDeploymentsWithChartRefs(t *testing. Status: "Succeeded", Type: 3, }, { - ChartId: 1, + ChartRefId: 1, ChartVersion: "v1.0.1", ChartType: "Deployment", EnvironmentId: 2, From 7b9be042eba4b731b582e5e62471a13e4410c9b9 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Thu, 25 Jan 2024 16:57:00 +0530 Subject: [PATCH 44/64] removed typo --- client/telemetry/TelemetryEventClient.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/telemetry/TelemetryEventClient.go b/client/telemetry/TelemetryEventClient.go index 5b34cd95c9..c806f42f7f 100644 --- a/client/telemetry/TelemetryEventClient.go +++ b/client/telemetry/TelemetryEventClient.go @@ -67,7 +67,7 @@ type TelemetryEventClient interface { SendTelemetryDashboardLoggedInEvent() error SendGenericTelemetryEvent(eventType string, prop map[string]interface{}) error SendSummaryEvent(eventType string) error -}w +} func NewTelemetryEventClientImpl(logger *zap.SugaredLogger, client *http.Client, clusterService cluster.ClusterService, K8sUtil *k8s.K8sServiceImpl, aCDAuthConfig *util3.ACDAuthConfig, userService user2.UserService, From 2aa56bfb3a0b415307217a3deacc5a402eec9376 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Thu, 25 Jan 2024 17:58:28 +0530 Subject: [PATCH 45/64] changes --- .../gitOps/git/GitOperationService.go | 18 ++++----- pkg/deployment/gitOps/git/GitService.go | 39 ++++--------------- pkg/deployment/gitOps/git/GitServiceAzure.go | 2 +- pkg/deployment/gitOps/git/GitServiceGithub.go | 6 ++- pkg/deployment/gitOps/git/GitServiceGitlab.go | 11 ++++-- pkg/deployment/gitOps/git/GitService_test.go | 3 +- pkg/deployment/gitOps/git/bean/bean.go | 14 +++++++ .../chartRef/ChartRefService.go | 14 ++++--- 8 files changed, 55 insertions(+), 52 deletions(-) diff --git a/pkg/deployment/gitOps/git/GitOperationService.go b/pkg/deployment/gitOps/git/GitOperationService.go index 0f9c9e3a18..70c88e3f82 100644 --- a/pkg/deployment/gitOps/git/GitOperationService.go +++ b/pkg/deployment/gitOps/git/GitOperationService.go @@ -34,7 +34,7 @@ type GitOperationService interface { GetRepoUrlByRepoName(repoName string) (string, error) GetClonedDir(chartDir, repoUrl string) (string, error) - CloneDir(repoUrl, chartDir string) (string, error) + CloneInDir(repoUrl, chartDir string) (string, error) } type GitOperationServiceImpl struct { @@ -91,6 +91,7 @@ func (impl *GitOperationServiceImpl) PushChartToGitRepo(gitOpsRepoName, referenc tempReferenceTemplateDir string, repoUrl string, userId int32) (err error) { chartDir := fmt.Sprintf("%s-%s", gitOpsRepoName, impl.chartTemplateService.GetDir()) clonedDir, err := impl.GetClonedDir(chartDir, repoUrl) + defer impl.chartTemplateService.CleanDir(clonedDir) if err != nil { impl.logger.Errorw("error in cloning repo", "url", repoUrl, "err", err) return err @@ -101,7 +102,7 @@ func (impl *GitOperationServiceImpl) PushChartToGitRepo(gitOpsRepoName, referenc return err } dir := filepath.Join(clonedDir, referenceTemplate, version) - pushChartToGit := true + performFirstCommitPush := true //if chart already exists don't overrides it by reference template if _, err := os.Stat(dir); os.IsNotExist(err) { @@ -128,12 +129,12 @@ func (impl *GitOperationServiceImpl) PushChartToGitRepo(gitOpsRepoName, referenc } } else { // chart exists on git, hence not performing first commit - pushChartToGit = false + performFirstCommitPush = false } } // if push needed, then only push - if pushChartToGit { + if performFirstCommitPush { userEmailId, userName := impl.gitOpsConfigReadService.GetUserEmailIdAndNameForGitOpsCommit(userId) commit, err := impl.gitFactory.GitService.CommitAndPushAllChanges(clonedDir, "first commit", userName, userEmailId) if err != nil { @@ -157,7 +158,6 @@ func (impl *GitOperationServiceImpl) PushChartToGitRepo(gitOpsRepoName, referenc impl.logger.Debugw("template committed", "url", repoUrl, "commit", commit) } - defer impl.chartTemplateService.CleanDir(clonedDir) return nil } @@ -176,6 +176,7 @@ func (impl *GitOperationServiceImpl) CreateReadmeInGitRepo(gitOpsRepoName string } _, err = impl.gitFactory.Client.CreateReadme(gitOpsConfig) if err != nil { + impl.logger.Errorw("error in creating readme", "err", err, "gitOpsRepoName", gitOpsRepoName, "userId", userId) return err } return nil @@ -330,9 +331,6 @@ func (impl *GitOperationServiceImpl) CommitValues(chartGitAttr *ChartConfig) (co impl.logger.Errorw("error in git commit", "err", err) return commitHash, commitTime, err } - if commitTime.IsZero() { - commitTime = time.Now() - } return commitHash, commitTime, nil } @@ -385,7 +383,7 @@ func (impl *GitOperationServiceImpl) GetRepoUrlByRepoName(repoName string) (stri func (impl *GitOperationServiceImpl) GetClonedDir(chartDir, repoUrl string) (string, error) { clonedDir := impl.gitFactory.GitService.GetCloneDirectory(chartDir) if _, err := os.Stat(clonedDir); os.IsNotExist(err) { - return impl.CloneDir(repoUrl, chartDir) + return impl.CloneInDir(repoUrl, chartDir) } else if err != nil { impl.logger.Errorw("error in cloning repo", "url", repoUrl, "err", err) return "", err @@ -393,7 +391,7 @@ func (impl *GitOperationServiceImpl) GetClonedDir(chartDir, repoUrl string) (str return clonedDir, nil } -func (impl *GitOperationServiceImpl) CloneDir(repoUrl, chartDir string) (string, error) { +func (impl *GitOperationServiceImpl) CloneInDir(repoUrl, chartDir string) (string, error) { clonedDir, err := impl.gitFactory.GitService.Clone(repoUrl, chartDir) if err != nil { impl.logger.Errorw("error in cloning repo", "url", repoUrl, "err", err) diff --git a/pkg/deployment/gitOps/git/GitService.go b/pkg/deployment/gitOps/git/GitService.go index 20105abe01..7577f2c8ae 100644 --- a/pkg/deployment/gitOps/git/GitService.go +++ b/pkg/deployment/gitOps/git/GitService.go @@ -20,6 +20,8 @@ package git import ( "context" "fmt" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git/adapter" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git/bean" "github.com/devtron-labs/devtron/util" "path/filepath" "time" @@ -122,18 +124,7 @@ func (factory *GitFactory) NewClientForValidation(gitOpsConfig *bean2.GitOpsConf defer func() { util.TriggerGitOpsMetrics("NewClientForValidation", "GitService", start, err) }() - cfg := &GitConfig{ - GitlabGroupId: gitOpsConfig.GitLabGroupId, - GitToken: gitOpsConfig.Token, - GitUserName: gitOpsConfig.Username, - GithubOrganization: gitOpsConfig.GitHubOrgId, - GitProvider: gitOpsConfig.Provider, - GitHost: gitOpsConfig.Host, - AzureToken: gitOpsConfig.Token, - AzureProject: gitOpsConfig.AzureProjectName, - BitbucketWorkspaceId: gitOpsConfig.BitBucketWorkspaceId, - BitbucketProjectKey: gitOpsConfig.BitBucketProjectKey, - } + cfg := adapter.ConvertGitOpsConfigToGitConfig(gitOpsConfig) gitService := NewGitServiceImpl(cfg, factory.logger, factory.gitCliUtil) //factory.GitService = GitService client, err := NewGitOpsClient(cfg, factory.logger, gitService) @@ -164,21 +155,7 @@ func NewGitFactory(logger *zap.SugaredLogger, gitOpsRepository repository.GitOps }, nil } -type GitConfig struct { - GitlabGroupId string //local - GitlabGroupPath string //local - GitToken string //not null // public - GitUserName string //not null // public - GithubOrganization string - GitProvider string // SUPPORTED VALUES GITHUB, GITLAB - GitHost string - AzureToken string - AzureProject string - BitbucketWorkspaceId string - BitbucketProjectKey string -} - -func GetGitConfig(gitOpsRepository repository.GitOpsConfigRepository) (*GitConfig, error) { +func GetGitConfig(gitOpsRepository repository.GitOpsConfigRepository) (*bean.GitConfig, error) { gitOpsConfig, err := gitOpsRepository.GetGitOpsConfigActive() if err != nil && err != pg.ErrNoRows { return nil, err @@ -187,13 +164,13 @@ func GetGitConfig(gitOpsRepository repository.GitOpsConfigRepository) (*GitConfi // cfg := &GitConfig{} // err := env.Parse(cfg) // return cfg, err - return &GitConfig{}, nil + return &bean.GitConfig{}, nil } if gitOpsConfig == nil || gitOpsConfig.Id == 0 { return nil, err } - cfg := &GitConfig{ + cfg := &bean.GitConfig{ GitlabGroupId: gitOpsConfig.GitLabGroupId, GitToken: gitOpsConfig.Token, GitUserName: gitOpsConfig.Username, @@ -208,7 +185,7 @@ func GetGitConfig(gitOpsRepository repository.GitOpsConfigRepository) (*GitConfi return cfg, err } -func NewGitOpsClient(config *GitConfig, logger *zap.SugaredLogger, gitService GitService) (GitClient, error) { +func NewGitOpsClient(config *bean.GitConfig, logger *zap.SugaredLogger, gitService GitService) (GitClient, error) { if config.GitProvider == GITLAB_PROVIDER { gitLabClient, err := NewGitLabClient(config, logger, gitService) return gitLabClient, err @@ -252,7 +229,7 @@ type GitServiceImpl struct { gitCliUtil *GitCliUtil } -func NewGitServiceImpl(config *GitConfig, logger *zap.SugaredLogger, GitCliUtil *GitCliUtil) *GitServiceImpl { +func NewGitServiceImpl(config *bean.GitConfig, logger *zap.SugaredLogger, GitCliUtil *GitCliUtil) *GitServiceImpl { auth := &http.BasicAuth{Password: config.GitToken, Username: config.GitUserName} return &GitServiceImpl{ Auth: auth, diff --git a/pkg/deployment/gitOps/git/GitServiceAzure.go b/pkg/deployment/gitOps/git/GitServiceAzure.go index b1fd155d1e..d474070f3d 100644 --- a/pkg/deployment/gitOps/git/GitServiceAzure.go +++ b/pkg/deployment/gitOps/git/GitServiceAzure.go @@ -233,7 +233,7 @@ func (impl GitAzureClient) CommitValues(config *ChartConfig, gitOpsConfig *bean2 } //gitPush.Commits commitId := "" - commitAuthorTime := time.Time{} + commitAuthorTime := time.Now() //default is current time, if found then will get updated accordingly if len(*push.Commits) > 0 { commitId = *(*push.Commits)[0].CommitId commitAuthorTime = (*push.Commits)[0].Author.Date.Time diff --git a/pkg/deployment/gitOps/git/GitServiceGithub.go b/pkg/deployment/gitOps/git/GitServiceGithub.go index 0969370cd4..2d4fc52fbf 100644 --- a/pkg/deployment/gitOps/git/GitServiceGithub.go +++ b/pkg/deployment/gitOps/git/GitServiceGithub.go @@ -191,7 +191,11 @@ func (impl GitHubClient) CommitValues(config *ChartConfig, gitOpsConfig *bean2.G impl.logger.Errorw("error in commit github", "err", err, "config", config) return "", time.Time{}, err } - return *c.SHA, *c.Commit.Author.Date, nil + commitTime = time.Now() // default is current time, if found then will get updated accordingly + if c != nil && c.Commit.Author != nil { + commitTime = *c.Commit.Author.Date + } + return *c.SHA, commitTime, nil } func (impl GitHubClient) GetRepoUrl(config *bean2.GitOpsConfigDto) (repoUrl string, err error) { diff --git a/pkg/deployment/gitOps/git/GitServiceGitlab.go b/pkg/deployment/gitOps/git/GitServiceGitlab.go index bf4a69cc63..12eb11e85e 100644 --- a/pkg/deployment/gitOps/git/GitServiceGitlab.go +++ b/pkg/deployment/gitOps/git/GitServiceGitlab.go @@ -3,6 +3,7 @@ package git import ( "fmt" bean2 "github.com/devtron-labs/devtron/api/bean" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git/bean" "github.com/xanzy/go-gitlab" "go.uber.org/zap" "net/url" @@ -13,12 +14,12 @@ import ( type GitLabClient struct { client *gitlab.Client - config *GitConfig + config *bean.GitConfig logger *zap.SugaredLogger gitService GitService } -func NewGitLabClient(config *GitConfig, logger *zap.SugaredLogger, gitService GitService) (GitClient, error) { +func NewGitLabClient(config *bean.GitConfig, logger *zap.SugaredLogger, gitService GitService) (GitClient, error) { gitLabClient, err := CreateGitlabClient(config.GitHost, config.GitToken) if err != nil { logger.Errorw("error in creating gitlab client", "err", err) @@ -275,5 +276,9 @@ func (impl GitLabClient) CommitValues(config *ChartConfig, gitOpsConfig *bean2.G if err != nil { return "", time.Time{}, err } - return c.ID, *c.AuthoredDate, err + commitTime = time.Now() //default is current time, if found then will get updated accordingly + if c != nil { + commitTime = *c.AuthoredDate + } + return c.ID, commitTime, err } diff --git a/pkg/deployment/gitOps/git/GitService_test.go b/pkg/deployment/gitOps/git/GitService_test.go index f23da2212b..3726bfdfcc 100644 --- a/pkg/deployment/gitOps/git/GitService_test.go +++ b/pkg/deployment/gitOps/git/GitService_test.go @@ -3,13 +3,14 @@ package git import ( "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/util" + bean2 "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git/bean" "testing" ) func getTestGithubClient() GitHubClient { logger, err := util.NewSugardLogger() gitCliUtl := NewGitCliUtil(logger) - gitService := NewGitServiceImpl(&GitConfig{GitToken: "", GitUserName: "nishant"}, logger, gitCliUtl) + gitService := NewGitServiceImpl(&bean2.GitConfig{GitToken: "", GitUserName: "nishant"}, logger, gitCliUtl) githubClient, err := NewGithubClient("", "", "test-org", logger, gitService) if err != nil { diff --git a/pkg/deployment/gitOps/git/bean/bean.go b/pkg/deployment/gitOps/git/bean/bean.go index 361b0ec78c..c806e55251 100644 --- a/pkg/deployment/gitOps/git/bean/bean.go +++ b/pkg/deployment/gitOps/git/bean/bean.go @@ -5,3 +5,17 @@ type ChartProxyReqDto struct { AppName string `json:"appName,omitempty"` UserId int32 `json:"-"` } + +type GitConfig struct { + GitlabGroupId string //local + GitlabGroupPath string //local + GitToken string //not null // public + GitUserName string //not null // public + GithubOrganization string + GitProvider string // SUPPORTED VALUES GITHUB, GITLAB + GitHost string + AzureToken string + AzureProject string + BitbucketWorkspaceId string + BitbucketProjectKey string +} diff --git a/pkg/deployment/manifest/deploymentTemplate/chartRef/ChartRefService.go b/pkg/deployment/manifest/deploymentTemplate/chartRef/ChartRefService.go index 58964b0876..9db0e2ade5 100644 --- a/pkg/deployment/manifest/deploymentTemplate/chartRef/ChartRefService.go +++ b/pkg/deployment/manifest/deploymentTemplate/chartRef/ChartRefService.go @@ -24,8 +24,12 @@ import ( ) type ChartRefService interface { - //below methods are for getting data from db + ChartRefDbReadService + CustomChartService + ChartRefFileOpService +} +type ChartRefDbReadService interface { GetDefault() (*bean.ChartRefDto, error) GetAll() ([]*bean.ChartRefDto, error) GetAllChartMetadata() (map[string]bean.ChartRefMetaData, error) @@ -35,15 +39,15 @@ type ChartRefService interface { ChartRefAutocomplete() ([]*bean.ChartRefAutocompleteDto, error) CheckChartExists(chartRefId int) error ChartRefIdsCompatible(oldChartRefId int, newChartRefId int) (bool, string, string) +} - //below methods are for custom chart - +type CustomChartService interface { SaveCustomChart(req *bean.CustomChartRefDto) error FetchCustomChartsInfo() ([]*bean.ChartDto, error) ValidateCustomChartUploadedFileFormat(fileName string) error +} - //below methods are for chart file data actions - +type ChartRefFileOpService interface { GetAppOverrideForDefaultTemplate(chartRefId int) (map[string]interface{}, string, error) JsonSchemaExtractFromFile(chartRefId int) (map[string]interface{}, string, error) GetSchemaAndReadmeForTemplateByChartRefId(chartRefId int) ([]byte, []byte, error) From 60fcf0a52eb4f82d153ba4df7e7bb460e32d700a Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Thu, 25 Jan 2024 17:59:16 +0530 Subject: [PATCH 46/64] changes --- pkg/deployment/gitOps/git/adapter/adapter.go | 21 ++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 pkg/deployment/gitOps/git/adapter/adapter.go diff --git a/pkg/deployment/gitOps/git/adapter/adapter.go b/pkg/deployment/gitOps/git/adapter/adapter.go new file mode 100644 index 0000000000..62b7f9c964 --- /dev/null +++ b/pkg/deployment/gitOps/git/adapter/adapter.go @@ -0,0 +1,21 @@ +package adapter + +import ( + bean2 "github.com/devtron-labs/devtron/api/bean" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git/bean" +) + +func ConvertGitOpsConfigToGitConfig(dto *bean2.GitOpsConfigDto) *bean.GitConfig { + return &bean.GitConfig{ + GitlabGroupId: dto.GitLabGroupId, + GitToken: dto.Token, + GitUserName: dto.Username, + GithubOrganization: dto.GitHubOrgId, + GitProvider: dto.Provider, + GitHost: dto.Host, + AzureToken: dto.Token, + AzureProject: dto.AzureProjectName, + BitbucketWorkspaceId: dto.BitBucketWorkspaceId, + BitbucketProjectKey: dto.BitBucketProjectKey, + } +} From a114cfc6788ef3137b01daae9e6e640f22974c83 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Wed, 31 Jan 2024 15:11:28 +0530 Subject: [PATCH 47/64] extracted trigger cd, nats subscriptions from wfDAGExec service --- App.go | 42 +- Wire.go | 5 +- api/bean/ValuesOverrideRequest.go | 17 - .../pipelineConfig/CdWorfkflowRepository.go | 12 + pkg/app/AppService.go | 6 +- .../status/PipelineStatusTimelineService.go | 31 + pkg/auth/user/bean/bean.go | 1 + pkg/bean/app.go | 32 +- .../trigger/devtronApps/TriggerService.go | 648 +++++++++++ .../trigger/devtronApps/bean/bean.go | 50 + .../trigger/devtronApps/helper/helper.go | 54 + .../wire_devtronAppsDeployTrigger.go | 8 + pkg/deployment/wire_deployment.go | 2 + .../CentralEventProcessorService.go | 44 + pkg/eventProcessor/bean/workflowEventBean.go | 27 + .../in/WorkflowEventProcessorService.go | 249 ++++ .../in/wire_eventProcessorIn.go | 7 + pkg/eventProcessor/wire_eventProcessor.go | 11 + .../AppDeploymentTypeChangeManager.go | 29 +- pkg/pipeline/CdHandler.go | 4 +- pkg/pipeline/WorkflowDagExecutor.go | 1012 +---------------- pkg/workflow/cd/CdWorkflowCommonService.go | 132 +++ pkg/workflow/cd/CdWorkflowRunnerService.go | 35 + pkg/workflow/cd/CdWorkflowService.go | 45 + pkg/workflow/cd/adapter/adapter.go | 48 + pkg/workflow/cd/bean/bean.go | 37 + pkg/workflow/cd/wire_workflowCd.go | 12 + util/argo/ArgoUserService.go | 14 + util/argo/HelmUserService.go | 6 + wire_gen.go | 23 +- 30 files changed, 1593 insertions(+), 1050 deletions(-) create mode 100644 pkg/deployment/trigger/devtronApps/TriggerService.go create mode 100644 pkg/deployment/trigger/devtronApps/bean/bean.go create mode 100644 pkg/deployment/trigger/devtronApps/helper/helper.go create mode 100644 pkg/deployment/trigger/devtronApps/wire_devtronAppsDeployTrigger.go create mode 100644 pkg/eventProcessor/CentralEventProcessorService.go create mode 100644 pkg/eventProcessor/bean/workflowEventBean.go create mode 100644 pkg/eventProcessor/in/WorkflowEventProcessorService.go create mode 100644 pkg/eventProcessor/in/wire_eventProcessorIn.go create mode 100644 pkg/eventProcessor/wire_eventProcessor.go create mode 100644 pkg/workflow/cd/CdWorkflowCommonService.go create mode 100644 pkg/workflow/cd/CdWorkflowRunnerService.go create mode 100644 pkg/workflow/cd/CdWorkflowService.go create mode 100644 pkg/workflow/cd/adapter/adapter.go create mode 100644 pkg/workflow/cd/bean/bean.go create mode 100644 pkg/workflow/cd/wire_workflowCd.go diff --git a/App.go b/App.go index 0337134d49..279023f3ab 100644 --- a/App.go +++ b/App.go @@ -22,6 +22,7 @@ import ( "crypto/tls" "fmt" "github.com/devtron-labs/common-lib/middlewares" + "github.com/devtron-labs/devtron/pkg/eventProcessor" "log" "net/http" "os" @@ -45,14 +46,15 @@ import ( ) type App struct { - MuxRouter *router.MuxRouter - Logger *zap.SugaredLogger - SSE *sse.SSE - Enforcer *casbin.SyncedEnforcer - server *http.Server - db *pg.DB - pubsubClient *pubsub.PubSubClientServiceImpl - posthogClient *telemetry.PosthogClient + MuxRouter *router.MuxRouter + Logger *zap.SugaredLogger + SSE *sse.SSE + Enforcer *casbin.SyncedEnforcer + server *http.Server + db *pg.DB + pubsubClient *pubsub.PubSubClientServiceImpl + posthogClient *telemetry.PosthogClient + centralEventProcessor *eventProcessor.CentralEventProcessor // used for local dev only serveTls bool sessionManager2 *authMiddleware.SessionManager @@ -69,21 +71,23 @@ func NewApp(router *router.MuxRouter, sessionManager2 *authMiddleware.SessionManager, posthogClient *telemetry.PosthogClient, loggingMiddleware util.LoggingMiddleware, + centralEventProcessor *eventProcessor.CentralEventProcessor, ) *App { //check argo connection //todo - check argo-cd version on acd integration installation app := &App{ - MuxRouter: router, - Logger: Logger, - SSE: sse, - Enforcer: enforcer, - db: db, - pubsubClient: pubsubClient, - serveTls: false, - sessionManager2: sessionManager2, - posthogClient: posthogClient, - OtelTracingService: otel.NewOtelTracingServiceImpl(Logger), - loggingMiddleware: loggingMiddleware, + MuxRouter: router, + Logger: Logger, + SSE: sse, + Enforcer: enforcer, + db: db, + pubsubClient: pubsubClient, + serveTls: false, + sessionManager2: sessionManager2, + posthogClient: posthogClient, + OtelTracingService: otel.NewOtelTracingServiceImpl(Logger), + loggingMiddleware: loggingMiddleware, + centralEventProcessor: centralEventProcessor, } return app } diff --git a/Wire.go b/Wire.go index 1a26330f4d..d004fe107d 100644 --- a/Wire.go +++ b/Wire.go @@ -104,6 +104,7 @@ import ( "github.com/devtron-labs/devtron/pkg/devtronResource" repository9 "github.com/devtron-labs/devtron/pkg/devtronResource/repository" "github.com/devtron-labs/devtron/pkg/dockerRegistry" + "github.com/devtron-labs/devtron/pkg/eventProcessor" "github.com/devtron-labs/devtron/pkg/generateManifest" "github.com/devtron-labs/devtron/pkg/git" "github.com/devtron-labs/devtron/pkg/gitops" @@ -127,6 +128,7 @@ import ( "github.com/devtron-labs/devtron/pkg/variables" "github.com/devtron-labs/devtron/pkg/variables/parsers" repository10 "github.com/devtron-labs/devtron/pkg/variables/repository" + "github.com/devtron-labs/devtron/pkg/workflow/cd" util2 "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/argo" cron2 "github.com/devtron-labs/devtron/util/cron" @@ -162,7 +164,8 @@ func InitializeApp() (*App, error) { webhookHelm.WebhookHelmWireSet, terminal.TerminalWireSet, deployment2.DeploymentWireSet, - + eventProcessor.EventProcessorWireSet, + cd.CdWorkflowWireSet, // -------wireset end ---------- //------- gitSensor.GetConfig, diff --git a/api/bean/ValuesOverrideRequest.go b/api/bean/ValuesOverrideRequest.go index 1e1e56e5a4..b7374ca7f7 100644 --- a/api/bean/ValuesOverrideRequest.go +++ b/api/bean/ValuesOverrideRequest.go @@ -21,7 +21,6 @@ import ( "encoding/json" "github.com/devtron-labs/devtron/internal/sql/models" "github.com/devtron-labs/devtron/pkg/pipeline/repository" - "time" ) type WorkflowType string @@ -81,27 +80,11 @@ type BulkCdDeployEvent struct { UserId int32 `json:"userId"` } -type AsyncCdDeployEvent struct { - ValuesOverrideRequest *ValuesOverrideRequest `json:"valuesOverrideRequest"` - TriggeredAt time.Time `json:"triggeredAt"` - TriggeredBy int32 `json:"triggeredBy"` -} - type ReleaseStatusUpdateRequest struct { RequestId string `json:"requestId"` NewStatus models.ChartStatus `json:"newStatus"` } -type TriggerEvent struct { - PerformChartPush bool - PerformDeploymentOnCluster bool - GetManifestInResponse bool - DeploymentAppType string - ManifestStorageType string - TriggeredBy int32 - TriggerdAt time.Time -} - type ArtifactsListFilterOptions struct { //list filter data Limit int diff --git a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go index ed6be1c69f..8614d32bb4 100644 --- a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go @@ -29,6 +29,7 @@ import ( repository2 "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/sql" + util4 "github.com/devtron-labs/devtron/util" "github.com/go-pg/pg" "go.opentelemetry.io/otel" "go.uber.org/zap" @@ -186,6 +187,17 @@ type CdWorkflowRunner struct { sql.AuditLog } +// TODO: move from here to adapter +func GetTriggerMetricsFromRunnerObj(runner *CdWorkflowRunner) util4.CDMetrics { + return util4.CDMetrics{ + AppName: runner.CdWorkflow.Pipeline.DeploymentAppName, + Status: runner.Status, + DeploymentType: runner.CdWorkflow.Pipeline.DeploymentAppType, + EnvironmentName: runner.CdWorkflow.Pipeline.Environment.Name, + Time: time.Since(runner.StartedOn).Seconds() - time.Since(runner.FinishedOn).Seconds(), + } +} + func (c *CdWorkflowRunner) IsExternalRun() bool { var isExtCluster bool if c.WorkflowType == WorkflowTypePre { diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index 62a158d4af..10e61a0a28 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -26,6 +26,7 @@ import ( "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" bean3 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/bean" + bean4 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "io/ioutil" "net/url" "os" @@ -40,7 +41,6 @@ import ( "github.com/devtron-labs/common-lib/utils/k8s/health" status2 "github.com/devtron-labs/devtron/pkg/app/status" repository4 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" - bean2 "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/resourceQualifiers" "github.com/devtron-labs/devtron/pkg/variables" _ "github.com/devtron-labs/devtron/pkg/variables/repository" @@ -500,7 +500,7 @@ func (impl *AppServiceImpl) CheckIfPipelineUpdateEventIsValid(argoAppName, gitHa return isValid, pipeline, cdWfr, pipelineOverride, err } //getting latest pipelineOverride for app (by appId and envId) - pipelineOverride, err = impl.pipelineOverrideRepository.FindLatestByAppIdAndEnvId(pipeline.AppId, pipeline.EnvironmentId, bean2.ArgoCd) + pipelineOverride, err = impl.pipelineOverrideRepository.FindLatestByAppIdAndEnvId(pipeline.AppId, pipeline.EnvironmentId, bean4.ArgoCd) if err != nil { impl.logger.Errorw("error in getting latest pipelineOverride by appId and envId", "err", err, "appId", pipeline.AppId, "envId", pipeline.EnvironmentId) return isValid, pipeline, cdWfr, pipelineOverride, err @@ -1257,5 +1257,5 @@ func (impl *AppServiceImpl) GetGitOpsRepoPrefix() string { func (impl *AppServiceImpl) IsDevtronAsyncInstallModeEnabled(deploymentAppType string) bool { return impl.appStatusConfig.EnableAsyncInstallDevtronChart && - deploymentAppType == bean2.Helm + deploymentAppType == bean4.Helm } diff --git a/pkg/app/status/PipelineStatusTimelineService.go b/pkg/app/status/PipelineStatusTimelineService.go index f85a35e30f..b3748b79f6 100644 --- a/pkg/app/status/PipelineStatusTimelineService.go +++ b/pkg/app/status/PipelineStatusTimelineService.go @@ -22,6 +22,8 @@ type PipelineStatusTimelineService interface { GetArgoAppSyncStatus(cdWfrId int) bool GetArgoAppSyncStatusForAppStore(installedAppVersionHistoryId int) bool SaveTimelines(timeline []*pipelineConfig.PipelineStatusTimeline, tx *pg.Tx) error + + MarkPipelineStatusTimelineFailed(cdWfrId int, statusDetailMessage string) error } type PipelineStatusTimelineServiceImpl struct { @@ -401,3 +403,32 @@ func (impl *PipelineStatusTimelineServiceImpl) SaveTimelines(timeline []*pipelin } return err } + +func (impl *PipelineStatusTimelineServiceImpl) MarkPipelineStatusTimelineFailed(cdWfrId int, statusDetailMessage string) error { + //creating cd pipeline status timeline for deployment failed + terminalStatusExists, timelineErr := impl.pipelineStatusTimelineRepository.CheckIfTerminalStatusTimelinePresentByWfrId(cdWfrId) + if timelineErr != nil { + impl.logger.Errorw("error in checking if terminal status timeline exists by wfrId", "err", timelineErr, "wfrId", cdWfrId) + return timelineErr + } + if !terminalStatusExists { + timeline := &pipelineConfig.PipelineStatusTimeline{ + CdWorkflowRunnerId: cdWfrId, + Status: pipelineConfig.TIMELINE_STATUS_DEPLOYMENT_FAILED, + StatusDetail: statusDetailMessage, + StatusTime: time.Now(), + AuditLog: sql.AuditLog{ + CreatedBy: 1, + CreatedOn: time.Now(), + UpdatedBy: 1, + UpdatedOn: time.Now(), + }, + } + impl.logger.Infow("marking pipeline deployment failed", "cdWfrId", cdWfrId, "statusDetail", statusDetailMessage) + timelineErr = impl.SaveTimeline(timeline, nil, false) + if timelineErr != nil { + impl.logger.Errorw("error in creating timeline status for deployment fail", "err", timelineErr, "timeline", timeline) + } + } + return nil +} diff --git a/pkg/auth/user/bean/bean.go b/pkg/auth/user/bean/bean.go index b4d7ea9b1a..d26e0a48eb 100644 --- a/pkg/auth/user/bean/bean.go +++ b/pkg/auth/user/bean/bean.go @@ -3,6 +3,7 @@ package bean type RoleType string const ( + SYSTEM_USER_ID = 1 PROJECT_TYPE = "team" ENV_TYPE = "environment" APP_TYPE = "app" diff --git a/pkg/bean/app.go b/pkg/bean/app.go index 275b051f76..70e253fbd5 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -26,6 +26,7 @@ import ( repository2 "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/pkg/chartRepo/repository" + bean3 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/repository" "time" @@ -662,12 +663,12 @@ const ( ) type DeploymentAppTypeChangeRequest struct { - EnvId int `json:"envId,omitempty" validate:"required"` - DesiredDeploymentType DeploymentType `json:"desiredDeploymentType,omitempty" validate:"required"` - ExcludeApps []int `json:"excludeApps"` - IncludeApps []int `json:"includeApps"` - AutoTriggerDeployment bool `json:"autoTriggerDeployment"` - UserId int32 `json:"-"` + EnvId int `json:"envId,omitempty" validate:"required"` + DesiredDeploymentType bean3.DeploymentType `json:"desiredDeploymentType,omitempty" validate:"required"` + ExcludeApps []int `json:"excludeApps"` + IncludeApps []int `json:"includeApps"` + AutoTriggerDeployment bool `json:"autoTriggerDeployment"` + UserId int32 `json:"-"` } type DeploymentChangeStatus struct { @@ -682,7 +683,7 @@ type DeploymentChangeStatus struct { type DeploymentAppTypeChangeResponse struct { EnvId int `json:"envId,omitempty"` - DesiredDeploymentType DeploymentType `json:"desiredDeploymentType,omitempty"` + DesiredDeploymentType bean3.DeploymentType `json:"desiredDeploymentType,omitempty"` SuccessfulPipelines []*DeploymentChangeStatus `json:"successfulPipelines"` FailedPipelines []*DeploymentChangeStatus `json:"failedPipelines"` TriggeredPipelines []*CdPipelineTrigger `json:"-"` // Disabling auto-trigger until bulk trigger API is fixed @@ -693,23 +694,6 @@ type CdPipelineTrigger struct { PipelineId int `json:"pipelineId"` } -type DeploymentType = string - -const ( - Helm DeploymentType = "helm" - ArgoCd DeploymentType = "argo_cd" - ManifestDownload DeploymentType = "manifest_download" - GitOpsWithoutDeployment DeploymentType = "git_ops_without_deployment" -) - -func IsAcdApp(deploymentType string) bool { - return deploymentType == ArgoCd -} - -func IsHelmApp(deploymentType string) bool { - return deploymentType == Helm -} - type Status string const ( diff --git a/pkg/deployment/trigger/devtronApps/TriggerService.go b/pkg/deployment/trigger/devtronApps/TriggerService.go new file mode 100644 index 0000000000..455985b8da --- /dev/null +++ b/pkg/deployment/trigger/devtronApps/TriggerService.go @@ -0,0 +1,648 @@ +package devtronApps + +import ( + "context" + "encoding/json" + "errors" + "fmt" + application3 "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" + "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" + pubsub "github.com/devtron-labs/common-lib/pubsub-lib" + util5 "github.com/devtron-labs/common-lib/utils/k8s" + bean3 "github.com/devtron-labs/devtron/api/bean" + bean6 "github.com/devtron-labs/devtron/api/helm-app/bean" + "github.com/devtron-labs/devtron/api/helm-app/gRPC" + client2 "github.com/devtron-labs/devtron/api/helm-app/service" + "github.com/devtron-labs/devtron/client/argocdServer" + application2 "github.com/devtron-labs/devtron/client/argocdServer/application" + client "github.com/devtron-labs/devtron/client/events" + "github.com/devtron-labs/devtron/internal/middleware" + repository3 "github.com/devtron-labs/devtron/internal/sql/repository" + "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/sql/repository/security" + "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/app" + bean4 "github.com/devtron-labs/devtron/pkg/app/bean" + "github.com/devtron-labs/devtron/pkg/app/status" + bean2 "github.com/devtron-labs/devtron/pkg/bean" + chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" + repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" + bean5 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/bean" + "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" + "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/helper" + "github.com/devtron-labs/devtron/pkg/pipeline/repository" + "github.com/devtron-labs/devtron/pkg/sql" + "github.com/devtron-labs/devtron/pkg/workflow/cd" + util2 "github.com/devtron-labs/devtron/util/event" + "github.com/go-pg/pg" + "go.opentelemetry.io/otel" + "go.uber.org/zap" + "google.golang.org/grpc/codes" + status2 "google.golang.org/grpc/status" + "k8s.io/helm/pkg/proto/hapi/chart" + "path" + "strings" + "time" +) + +type TriggerService interface { + TriggerPipeline(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, + builtChartPath string, triggerEvent bean.TriggerEvent, ctx context.Context) (releaseNo int, manifest []byte, err error) +} + +type TriggerServiceImpl struct { + logger *zap.SugaredLogger + cdWorkflowCommonService cd.CdWorkflowCommonService + gitOpsManifestPushService app.GitOpsPushService + argoK8sClient argocdServer.ArgoK8sClient + ACDConfig *argocdServer.ACDConfig + acdClient application2.ServiceClient + argoClientWrapperService argocdServer.ArgoClientWrapperService + pipelineStatusTimelineService status.PipelineStatusTimelineService + chartTemplateService util.ChartTemplateService + appService app.AppService + eventFactory client.EventFactory + eventClient client.EventClient + + helmAppService client2.HelmAppService + + helmAppClient gRPC.HelmAppClient //TODO refactoring: use helm app service instead + + ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository + imageScanHistoryRepository security.ImageScanHistoryRepository + imageScanDeployInfoRepository security.ImageScanDeployInfoRepository + pipelineRepository pipelineConfig.PipelineRepository + pipelineOverrideRepository chartConfig.PipelineOverrideRepository + manifestPushConfigRepository repository.ManifestPushConfigRepository + chartRepository chartRepoRepository.ChartRepository + envRepository repository2.EnvironmentRepository +} + +func NewTriggerServiceImpl(logger *zap.SugaredLogger, cdWorkflowCommonService cd.CdWorkflowCommonService, + gitOpsManifestPushService app.GitOpsPushService, + argoK8sClient argocdServer.ArgoK8sClient, + ACDConfig *argocdServer.ACDConfig, + acdClient application2.ServiceClient, + argoClientWrapperService argocdServer.ArgoClientWrapperService, + pipelineStatusTimelineService status.PipelineStatusTimelineService, + chartTemplateService util.ChartTemplateService, + appService app.AppService, + helmAppService client2.HelmAppService, + helmAppClient gRPC.HelmAppClient, + eventFactory client.EventFactory, + eventClient client.EventClient, + ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, + imageScanHistoryRepository security.ImageScanHistoryRepository, + imageScanDeployInfoRepository security.ImageScanDeployInfoRepository, + pipelineRepository pipelineConfig.PipelineRepository, + pipelineOverrideRepository chartConfig.PipelineOverrideRepository, + manifestPushConfigRepository repository.ManifestPushConfigRepository, + chartRepository chartRepoRepository.ChartRepository, + envRepository repository2.EnvironmentRepository) *TriggerServiceImpl { + return &TriggerServiceImpl{ + logger: logger, + cdWorkflowCommonService: cdWorkflowCommonService, + gitOpsManifestPushService: gitOpsManifestPushService, + argoK8sClient: argoK8sClient, + ACDConfig: ACDConfig, + acdClient: acdClient, + argoClientWrapperService: argoClientWrapperService, + pipelineStatusTimelineService: pipelineStatusTimelineService, + chartTemplateService: chartTemplateService, + appService: appService, + helmAppService: helmAppService, + eventClient: eventClient, + eventFactory: eventFactory, + + helmAppClient: helmAppClient, + + ciPipelineMaterialRepository: ciPipelineMaterialRepository, + imageScanHistoryRepository: imageScanHistoryRepository, + imageScanDeployInfoRepository: imageScanDeployInfoRepository, + pipelineRepository: pipelineRepository, + pipelineOverrideRepository: pipelineOverrideRepository, + manifestPushConfigRepository: manifestPushConfigRepository, + chartRepository: chartRepository, + envRepository: envRepository, + } +} + +func (impl *TriggerServiceImpl) TriggerPipeline(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, triggerEvent bean.TriggerEvent, ctx context.Context) (releaseNo int, manifest []byte, err error) { + isRequestValid, err := helper.ValidateTriggerEvent(triggerEvent) + if !isRequestValid { + return releaseNo, manifest, err + } + + if triggerEvent.PerformChartPush { + //update workflow runner status, used in app workflow view + err = impl.cdWorkflowCommonService.UpdateCDWorkflowRunnerStatus(ctx, overrideRequest, triggerEvent.TriggerdAt, pipelineConfig.WorkflowInProgress, "") + if err != nil { + impl.logger.Errorw("error in updating the workflow runner status, createHelmAppForCdPipeline", "err", err) + return releaseNo, manifest, err + } + manifestPushTemplate, err := impl.BuildManifestPushTemplate(overrideRequest, valuesOverrideResponse, builtChartPath, &manifest) + if err != nil { + impl.logger.Errorw("error in building manifest push template", "err", err) + return releaseNo, manifest, err + } + manifestPushService := impl.GetManifestPushService(triggerEvent) + manifestPushResponse := manifestPushService.PushChart(manifestPushTemplate, ctx) + if manifestPushResponse.Error != nil { + impl.logger.Errorw("Error in pushing manifest to git", "err", err, "git_repo_url", manifestPushTemplate.RepoUrl) + return releaseNo, manifest, manifestPushResponse.Error + } + pipelineOverrideUpdateRequest := &chartConfig.PipelineOverride{ + Id: valuesOverrideResponse.PipelineOverride.Id, + GitHash: manifestPushResponse.CommitHash, + CommitTime: manifestPushResponse.CommitTime, + EnvConfigOverrideId: valuesOverrideResponse.EnvOverride.Id, + PipelineOverrideValues: valuesOverrideResponse.ReleaseOverrideJSON, + PipelineId: overrideRequest.PipelineId, + CiArtifactId: overrideRequest.CiArtifactId, + PipelineMergedValues: valuesOverrideResponse.MergedValues, + AuditLog: sql.AuditLog{UpdatedOn: triggerEvent.TriggerdAt, UpdatedBy: overrideRequest.UserId}, + } + _, span := otel.Tracer("orchestrator").Start(ctx, "pipelineOverrideRepository.Update") + err = impl.pipelineOverrideRepository.Update(pipelineOverrideUpdateRequest) + span.End() + } + + if triggerEvent.PerformDeploymentOnCluster { + err = impl.DeployApp(overrideRequest, valuesOverrideResponse, triggerEvent.TriggerdAt, ctx) + if err != nil { + impl.logger.Errorw("error in deploying app", "err", err) + return releaseNo, manifest, err + } + } + + go impl.WriteCDTriggerEvent(overrideRequest, valuesOverrideResponse.Artifact, valuesOverrideResponse.PipelineOverride.PipelineReleaseCounter, valuesOverrideResponse.PipelineOverride.Id) + + _, span := otel.Tracer("orchestrator").Start(ctx, "MarkImageScanDeployed") + _ = impl.MarkImageScanDeployed(overrideRequest.AppId, valuesOverrideResponse.EnvOverride.TargetEnvironment, valuesOverrideResponse.Artifact.ImageDigest, overrideRequest.ClusterId, valuesOverrideResponse.Artifact.ScanEnabled) + span.End() + + middleware.CdTriggerCounter.WithLabelValues(overrideRequest.AppName, overrideRequest.EnvName).Inc() + + return valuesOverrideResponse.PipelineOverride.PipelineReleaseCounter, manifest, nil + +} + +func (impl *TriggerServiceImpl) BuildManifestPushTemplate(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, manifest *[]byte) (*bean4.ManifestPushTemplate, error) { + + manifestPushTemplate := &bean4.ManifestPushTemplate{ + WorkflowRunnerId: overrideRequest.WfrId, + AppId: overrideRequest.AppId, + ChartRefId: valuesOverrideResponse.EnvOverride.Chart.ChartRefId, + EnvironmentId: valuesOverrideResponse.EnvOverride.Environment.Id, + UserId: overrideRequest.UserId, + PipelineOverrideId: valuesOverrideResponse.PipelineOverride.Id, + AppName: overrideRequest.AppName, + TargetEnvironmentName: valuesOverrideResponse.EnvOverride.TargetEnvironment, + BuiltChartPath: builtChartPath, + BuiltChartBytes: manifest, + MergedValues: valuesOverrideResponse.MergedValues, + } + + manifestPushConfig, err := impl.manifestPushConfigRepository.GetManifestPushConfigByAppIdAndEnvId(overrideRequest.AppId, overrideRequest.EnvId) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in fetching manifest push config from db", "err", err) + return manifestPushTemplate, err + } + + if manifestPushConfig != nil { + if manifestPushConfig.StorageType == bean2.ManifestStorageGit { + // need to implement for git repo push + // currently manifest push config doesn't have git push config. Gitops config is derived from charts, chart_env_config_override and chart_ref table + } + } else { + manifestPushTemplate.ChartReferenceTemplate = valuesOverrideResponse.EnvOverride.Chart.ReferenceTemplate + manifestPushTemplate.ChartName = valuesOverrideResponse.EnvOverride.Chart.ChartName + manifestPushTemplate.ChartVersion = valuesOverrideResponse.EnvOverride.Chart.ChartVersion + manifestPushTemplate.ChartLocation = valuesOverrideResponse.EnvOverride.Chart.ChartLocation + manifestPushTemplate.RepoUrl = valuesOverrideResponse.EnvOverride.Chart.GitRepoUrl + } + return manifestPushTemplate, err +} + +func (impl *TriggerServiceImpl) GetManifestPushService(triggerEvent bean.TriggerEvent) app.ManifestPushService { + var manifestPushService app.ManifestPushService + if triggerEvent.ManifestStorageType == bean2.ManifestStorageGit { + manifestPushService = impl.gitOpsManifestPushService + } + return manifestPushService +} + +func (impl *TriggerServiceImpl) DeployApp(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, triggeredAt time.Time, ctx context.Context) error { + + if util.IsAcdApp(overrideRequest.DeploymentAppType) { + _, span := otel.Tracer("orchestrator").Start(ctx, "DeployArgocdApp") + err := impl.DeployArgocdApp(overrideRequest, valuesOverrideResponse, triggeredAt, ctx) + span.End() + if err != nil { + impl.logger.Errorw("error in deploying app on argocd", "err", err) + return err + } + } else if util.IsHelmApp(overrideRequest.DeploymentAppType) { + _, span := otel.Tracer("orchestrator").Start(ctx, "createHelmAppForCdPipeline") + _, err := impl.createHelmAppForCdPipeline(overrideRequest, valuesOverrideResponse, triggeredAt, ctx) + span.End() + if err != nil { + impl.logger.Errorw("error in creating or updating helm application for cd pipeline", "err", err) + return err + } + } + return nil +} + +func (impl *TriggerServiceImpl) createHelmAppForCdPipeline(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, triggeredAt time.Time, ctx context.Context) (bool, error) { + + pipeline := valuesOverrideResponse.Pipeline + envOverride := valuesOverrideResponse.EnvOverride + mergeAndSave := valuesOverrideResponse.MergedValues + + chartMetaData := &chart.Metadata{ + Name: pipeline.App.AppName, + Version: envOverride.Chart.ChartVersion, + } + referenceTemplatePath := path.Join(bean5.RefChartDirPath, envOverride.Chart.ReferenceTemplate) + + if util.IsHelmApp(pipeline.DeploymentAppType) { + referenceChartByte := envOverride.Chart.ReferenceChart + // here updating reference chart into database. + if len(envOverride.Chart.ReferenceChart) == 0 { + refChartByte, err := impl.chartTemplateService.GetByteArrayRefChart(chartMetaData, referenceTemplatePath) + if err != nil { + impl.logger.Errorw("ref chart commit error on cd trigger", "err", err, "req", overrideRequest) + return false, err + } + ch := envOverride.Chart + ch.ReferenceChart = refChartByte + ch.UpdatedOn = time.Now() + ch.UpdatedBy = overrideRequest.UserId + err = impl.chartRepository.Update(ch) + if err != nil { + impl.logger.Errorw("chart update error", "err", err, "req", overrideRequest) + return false, err + } + referenceChartByte = refChartByte + } + + releaseName := pipeline.DeploymentAppName + cluster := envOverride.Environment.Cluster + bearerToken := cluster.Config[util5.BearerToken] + clusterConfig := &gRPC.ClusterConfig{ + ClusterName: cluster.ClusterName, + Token: bearerToken, + ApiServerUrl: cluster.ServerUrl, + InsecureSkipTLSVerify: cluster.InsecureSkipTlsVerify, + } + if cluster.InsecureSkipTlsVerify == false { + clusterConfig.KeyData = cluster.Config[util5.TlsKey] + clusterConfig.CertData = cluster.Config[util5.CertData] + clusterConfig.CaData = cluster.Config[util5.CertificateAuthorityData] + } + releaseIdentifier := &gRPC.ReleaseIdentifier{ + ReleaseName: releaseName, + ReleaseNamespace: envOverride.Namespace, + ClusterConfig: clusterConfig, + } + + if pipeline.DeploymentAppCreated { + req := &gRPC.UpgradeReleaseRequest{ + ReleaseIdentifier: releaseIdentifier, + ValuesYaml: mergeAndSave, + HistoryMax: impl.helmAppService.GetRevisionHistoryMaxValue(bean6.SOURCE_DEVTRON_APP), + ChartContent: &gRPC.ChartContent{Content: referenceChartByte}, + } + if impl.appService.IsDevtronAsyncInstallModeEnabled(bean.Helm) { + req.RunInCtx = true + } + // For cases where helm release was not found, kubelink will install the same configuration + updateApplicationResponse, err := impl.helmAppClient.UpdateApplication(ctx, req) + if err != nil { + impl.logger.Errorw("error in updating helm application for cd pipeline", "err", err) + if util.GetGRPCErrorDetailedMessage(err) == context.Canceled.Error() { + err = errors.New(pipelineConfig.NEW_DEPLOYMENT_INITIATED) + } + return false, err + } else { + impl.logger.Debugw("updated helm application", "response", updateApplicationResponse, "isSuccess", updateApplicationResponse.Success) + } + + } else { + + helmResponse, err := impl.helmInstallReleaseWithCustomChart(ctx, releaseIdentifier, referenceChartByte, mergeAndSave) + + // For connection related errors, no need to update the db + if err != nil && strings.Contains(err.Error(), "connection error") { + impl.logger.Errorw("error in helm install custom chart", "err", err) + return false, err + } + if util.GetGRPCErrorDetailedMessage(err) == context.Canceled.Error() { + err = errors.New(pipelineConfig.NEW_DEPLOYMENT_INITIATED) + } + + // IMP: update cd pipeline to mark deployment app created, even if helm install fails + // If the helm install fails, it still creates the app in failed state, so trying to + // re-create the app results in error from helm that cannot re-use name which is still in use + _, pgErr := impl.updatePipeline(pipeline, overrideRequest.UserId) + + if err != nil { + impl.logger.Errorw("error in helm install custom chart", "err", err) + + if pgErr != nil { + impl.logger.Errorw("failed to update deployment app created flag in pipeline table", "err", err) + } + return false, err + } + + if pgErr != nil { + impl.logger.Errorw("failed to update deployment app created flag in pipeline table", "err", err) + return false, err + } + + impl.logger.Debugw("received helm release response", "helmResponse", helmResponse, "isSuccess", helmResponse.Success) + } + + //update workflow runner status, used in app workflow view + err := impl.cdWorkflowCommonService.UpdateCDWorkflowRunnerStatus(ctx, overrideRequest, triggeredAt, pipelineConfig.WorkflowInProgress, "") + if err != nil { + impl.logger.Errorw("error in updating the workflow runner status, createHelmAppForCdPipeline", "err", err) + return false, err + } + } + return true, nil +} + +func (impl *TriggerServiceImpl) DeployArgocdApp(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, triggeredAt time.Time, ctx context.Context) error { + + impl.logger.Debugw("new pipeline found", "pipeline", valuesOverrideResponse.Pipeline) + _, span := otel.Tracer("orchestrator").Start(ctx, "createArgoApplicationIfRequired") + name, err := impl.createArgoApplicationIfRequired(overrideRequest.AppId, valuesOverrideResponse.EnvOverride, valuesOverrideResponse.Pipeline, overrideRequest.UserId) + span.End() + if err != nil { + impl.logger.Errorw("acd application create error on cd trigger", "err", err, "req", overrideRequest) + return err + } + impl.logger.Debugw("argocd application created", "name", name) + + _, span = otel.Tracer("orchestrator").Start(ctx, "updateArgoPipeline") + updateAppInArgocd, err := impl.updateArgoPipeline(valuesOverrideResponse.Pipeline, valuesOverrideResponse.EnvOverride, ctx) + span.End() + if err != nil { + impl.logger.Errorw("error in updating argocd app ", "err", err) + return err + } + syncTime := time.Now() + err = impl.argoClientWrapperService.SyncArgoCDApplicationIfNeededAndRefresh(ctx, valuesOverrideResponse.Pipeline.DeploymentAppName) + if err != nil { + impl.logger.Errorw("error in getting argo application with normal refresh", "argoAppName", valuesOverrideResponse.Pipeline.DeploymentAppName) + return fmt.Errorf("%s. err: %s", bean.ARGOCD_SYNC_ERROR, err.Error()) + } + if !impl.ACDConfig.ArgoCDAutoSyncEnabled { + timeline := &pipelineConfig.PipelineStatusTimeline{ + CdWorkflowRunnerId: overrideRequest.WfrId, + StatusTime: syncTime, + AuditLog: sql.AuditLog{ + CreatedBy: 1, + CreatedOn: time.Now(), + UpdatedBy: 1, + UpdatedOn: time.Now(), + }, + Status: pipelineConfig.TIMELINE_STATUS_ARGOCD_SYNC_COMPLETED, + StatusDetail: "argocd sync completed", + } + _, err, _ = impl.pipelineStatusTimelineService.SavePipelineStatusTimelineIfNotAlreadyPresent(overrideRequest.WfrId, timeline.Status, timeline, false) + if err != nil { + impl.logger.Errorw("error in saving pipeline status timeline", "err", err) + } + } + if updateAppInArgocd { + impl.logger.Debug("argo-cd successfully updated") + } else { + impl.logger.Debug("argo-cd failed to update, ignoring it") + } + return nil +} + +// update repoUrl, revision and argo app sync mode (auto/manual) if needed +func (impl *TriggerServiceImpl) updateArgoPipeline(pipeline *pipelineConfig.Pipeline, envOverride *chartConfig.EnvConfigOverride, ctx context.Context) (bool, error) { + if ctx == nil { + impl.logger.Errorw("err in syncing ACD, ctx is NULL", "pipelineName", pipeline.Name) + return false, nil + } + argoAppName := pipeline.DeploymentAppName + impl.logger.Infow("received payload, updateArgoPipeline", "appId", pipeline.AppId, "pipelineName", pipeline.Name, "envId", envOverride.TargetEnvironment, "argoAppName", argoAppName, "context", ctx) + argoApplication, err := impl.acdClient.Get(ctx, &application3.ApplicationQuery{Name: &argoAppName}) + if err != nil { + impl.logger.Errorw("no argo app exists", "app", argoAppName, "pipeline", pipeline.Name) + return false, err + } + //if status, ok:=status.FromError(err);ok{ + appStatus, _ := status2.FromError(err) + if appStatus.Code() == codes.OK { + impl.logger.Debugw("argo app exists", "app", argoAppName, "pipeline", pipeline.Name) + if argoApplication.Spec.Source.Path != envOverride.Chart.ChartLocation || argoApplication.Spec.Source.TargetRevision != "master" { + patchReq := v1alpha1.Application{Spec: v1alpha1.ApplicationSpec{Source: &v1alpha1.ApplicationSource{Path: envOverride.Chart.ChartLocation, RepoURL: envOverride.Chart.GitRepoUrl, TargetRevision: "master"}}} + reqbyte, err := json.Marshal(patchReq) + if err != nil { + impl.logger.Errorw("error in creating patch", "err", err) + } + reqString := string(reqbyte) + patchType := "merge" + _, err = impl.acdClient.Patch(ctx, &application3.ApplicationPatchRequest{Patch: &reqString, Name: &argoAppName, PatchType: &patchType}) + if err != nil { + impl.logger.Errorw("error in creating argo pipeline ", "name", pipeline.Name, "patch", string(reqbyte), "err", err) + return false, err + } + impl.logger.Debugw("pipeline update req ", "res", patchReq) + } else { + impl.logger.Debug("pipeline no need to update ") + } + err := impl.argoClientWrapperService.UpdateArgoCDSyncModeIfNeeded(ctx, argoApplication) + if err != nil { + impl.logger.Errorw("error in updating argocd sync mode", "err", err) + return false, err + } + return true, nil + } else if appStatus.Code() == codes.NotFound { + impl.logger.Errorw("argo app not found", "app", argoAppName, "pipeline", pipeline.Name) + return false, nil + } else { + impl.logger.Errorw("err in checking application on argoCD", "err", err, "pipeline", pipeline.Name) + return false, err + } +} + +func (impl *TriggerServiceImpl) createArgoApplicationIfRequired(appId int, envConfigOverride *chartConfig.EnvConfigOverride, pipeline *pipelineConfig.Pipeline, userId int32) (string, error) { + //repo has been registered while helm create + chart, err := impl.chartRepository.FindLatestChartForAppByAppId(appId) + if err != nil { + impl.logger.Errorw("no chart found ", "app", appId) + return "", err + } + envModel, err := impl.envRepository.FindById(envConfigOverride.TargetEnvironment) + if err != nil { + return "", err + } + argoAppName := pipeline.DeploymentAppName + if pipeline.DeploymentAppCreated { + return argoAppName, nil + } else { + //create + appNamespace := envConfigOverride.Namespace + if appNamespace == "" { + appNamespace = "default" + } + namespace := argocdServer.DevtronInstalationNs + + appRequest := &argocdServer.AppTemplate{ + ApplicationName: argoAppName, + Namespace: namespace, + TargetNamespace: appNamespace, + TargetServer: envModel.Cluster.ServerUrl, + Project: "default", + ValuesFile: impl.getValuesFileForEnv(envModel.Id), + RepoPath: chart.ChartLocation, + RepoUrl: chart.GitRepoUrl, + AutoSyncEnabled: impl.ACDConfig.ArgoCDAutoSyncEnabled, + } + argoAppName, err := impl.argoK8sClient.CreateAcdApp(appRequest, envModel.Cluster, argocdServer.ARGOCD_APPLICATION_TEMPLATE) + if err != nil { + return "", err + } + //update cd pipeline to mark deployment app created + _, err = impl.updatePipeline(pipeline, userId) + if err != nil { + impl.logger.Errorw("error in update cd pipeline for deployment app created or not", "err", err) + return "", err + } + return argoAppName, nil + } +} + +func (impl *TriggerServiceImpl) getValuesFileForEnv(environmentId int) string { + return fmt.Sprintf("_%d-values.yaml", environmentId) //-{envId}-values.yaml +} + +func (impl *TriggerServiceImpl) updatePipeline(pipeline *pipelineConfig.Pipeline, userId int32) (bool, error) { + err := impl.pipelineRepository.SetDeploymentAppCreatedInPipeline(true, pipeline.Id, userId) + if err != nil { + impl.logger.Errorw("error on updating cd pipeline for setting deployment app created", "err", err) + return false, err + } + return true, nil +} + +// helmInstallReleaseWithCustomChart performs helm install with custom chart +func (impl *TriggerServiceImpl) helmInstallReleaseWithCustomChart(ctx context.Context, releaseIdentifier *gRPC.ReleaseIdentifier, referenceChartByte []byte, valuesYaml string) (*gRPC.HelmInstallCustomResponse, error) { + + helmInstallRequest := gRPC.HelmInstallCustomRequest{ + ValuesYaml: valuesYaml, + ChartContent: &gRPC.ChartContent{Content: referenceChartByte}, + ReleaseIdentifier: releaseIdentifier, + } + if impl.appService.IsDevtronAsyncInstallModeEnabled(bean.Helm) { + helmInstallRequest.RunInCtx = true + } + // Request exec + return impl.helmAppClient.InstallReleaseWithCustomChart(ctx, &helmInstallRequest) +} + +func (impl *TriggerServiceImpl) WriteCDTriggerEvent(overrideRequest *bean3.ValuesOverrideRequest, artifact *repository3.CiArtifact, releaseId, pipelineOverrideId int) { + + event := impl.eventFactory.Build(util2.Trigger, &overrideRequest.PipelineId, overrideRequest.AppId, &overrideRequest.EnvId, util2.CD) + impl.logger.Debugw("event WriteCDTriggerEvent", "event", event) + event = impl.eventFactory.BuildExtraCDData(event, nil, pipelineOverrideId, bean3.CD_WORKFLOW_TYPE_DEPLOY) + _, evtErr := impl.eventClient.WriteNotificationEvent(event) + if evtErr != nil { + impl.logger.Errorw("CD trigger event not sent", "error", evtErr) + } + deploymentEvent := app.DeploymentEvent{ + ApplicationId: overrideRequest.AppId, + EnvironmentId: overrideRequest.EnvId, //check for production Environment + ReleaseId: releaseId, + PipelineOverrideId: pipelineOverrideId, + TriggerTime: time.Now(), + CiArtifactId: overrideRequest.CiArtifactId, + } + ciPipelineMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(artifact.PipelineId) + if err != nil { + impl.logger.Errorw("error in ") + } + materialInfoMap, mErr := artifact.ParseMaterialInfo() + if mErr != nil { + impl.logger.Errorw("material info map error", mErr) + return + } + for _, ciPipelineMaterial := range ciPipelineMaterials { + hash := materialInfoMap[ciPipelineMaterial.GitMaterial.Url] + pipelineMaterialInfo := &app.PipelineMaterialInfo{PipelineMaterialId: ciPipelineMaterial.Id, CommitHash: hash} + deploymentEvent.PipelineMaterials = append(deploymentEvent.PipelineMaterials, pipelineMaterialInfo) + } + impl.logger.Infow("triggering deployment event", "event", deploymentEvent) + err = impl.eventClient.WriteNatsEvent(pubsub.CD_SUCCESS, deploymentEvent) + if err != nil { + impl.logger.Errorw("error in writing cd trigger event", "err", err) + } +} + +func (impl *TriggerServiceImpl) MarkImageScanDeployed(appId int, envId int, imageDigest string, clusterId int, isScanEnabled bool) error { + impl.logger.Debugw("mark image scan deployed for normal app, from cd auto or manual trigger", "imageDigest", imageDigest) + executionHistory, err := impl.imageScanHistoryRepository.FindByImageDigest(imageDigest) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in fetching execution history", "err", err) + return err + } + if executionHistory == nil || executionHistory.Id == 0 { + impl.logger.Errorw("no execution history found for digest", "digest", imageDigest) + return fmt.Errorf("no execution history found for digest - %s", imageDigest) + } + impl.logger.Debugw("mark image scan deployed for normal app, from cd auto or manual trigger", "executionHistory", executionHistory) + var ids []int + ids = append(ids, executionHistory.Id) + + ot, err := impl.imageScanDeployInfoRepository.FetchByAppIdAndEnvId(appId, envId, []string{security.ScanObjectType_APP}) + + if err == pg.ErrNoRows && !isScanEnabled { + //ignoring if no rows are found and scan is disabled + return nil + } + + if err != nil && err != pg.ErrNoRows { + return err + } else if err == pg.ErrNoRows && isScanEnabled { + imageScanDeployInfo := &security.ImageScanDeployInfo{ + ImageScanExecutionHistoryId: ids, + ScanObjectMetaId: appId, + ObjectType: security.ScanObjectType_APP, + EnvId: envId, + ClusterId: clusterId, + AuditLog: sql.AuditLog{ + CreatedOn: time.Now(), + CreatedBy: 1, + UpdatedOn: time.Now(), + UpdatedBy: 1, + }, + } + impl.logger.Debugw("mark image scan deployed for normal app, from cd auto or manual trigger", "imageScanDeployInfo", imageScanDeployInfo) + err = impl.imageScanDeployInfoRepository.Save(imageScanDeployInfo) + if err != nil { + impl.logger.Errorw("error in creating deploy info", "err", err) + } + } else { + // Updating Execution history for Latest Deployment to fetch out security Vulnerabilities for latest deployed info + if isScanEnabled { + ot.ImageScanExecutionHistoryId = ids + } else { + arr := []int{-1} + ot.ImageScanExecutionHistoryId = arr + } + err = impl.imageScanDeployInfoRepository.Update(ot) + if err != nil { + impl.logger.Errorw("error in updating deploy info for latest deployed image", "err", err) + } + } + return err +} diff --git a/pkg/deployment/trigger/devtronApps/bean/bean.go b/pkg/deployment/trigger/devtronApps/bean/bean.go new file mode 100644 index 0000000000..8fd4c283ae --- /dev/null +++ b/pkg/deployment/trigger/devtronApps/bean/bean.go @@ -0,0 +1,50 @@ +package bean + +import ( + "context" + "github.com/devtron-labs/devtron/internal/sql/repository" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "time" +) + +const ( + ARGOCD_SYNC_ERROR = "error in syncing argoCD app" +) + +type TriggerEvent struct { + PerformChartPush bool + PerformDeploymentOnCluster bool + GetManifestInResponse bool + DeploymentAppType string + ManifestStorageType string + TriggeredBy int32 + TriggerdAt time.Time +} + +type TriggerRequest struct { + CdWf *pipelineConfig.CdWorkflow + Pipeline *pipelineConfig.Pipeline + Artifact *repository.CiArtifact + ApplyAuth bool + TriggeredBy int32 + RefCdWorkflowRunnerId int + TriggerContext +} + +type TriggerContext struct { + // Context is a context object to be passed to the pipeline trigger + // +optional + Context context.Context + // ReferenceId is a unique identifier for the workflow runner + // refer pipelineConfig.CdWorkflowRunner + ReferenceId *string +} + +type DeploymentType = string + +const ( + Helm DeploymentType = "helm" + ArgoCd DeploymentType = "argo_cd" + ManifestDownload DeploymentType = "manifest_download" + GitOpsWithoutDeployment DeploymentType = "git_ops_without_deployment" +) diff --git a/pkg/deployment/trigger/devtronApps/helper/helper.go b/pkg/deployment/trigger/devtronApps/helper/helper.go new file mode 100644 index 0000000000..a46b63f624 --- /dev/null +++ b/pkg/deployment/trigger/devtronApps/helper/helper.go @@ -0,0 +1,54 @@ +package helper + +import ( + errors3 "errors" + bean2 "github.com/devtron-labs/devtron/pkg/bean" + "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" + errors2 "github.com/juju/errors" + "time" +) + +func GetTriggerEvent(deploymentAppType string, triggeredAt time.Time, deployedBy int32) bean.TriggerEvent { + // trigger event will decide whether to perform GitOps or deployment for a particular deployment app type + triggerEvent := bean.TriggerEvent{ + TriggeredBy: deployedBy, + TriggerdAt: triggeredAt, + } + switch deploymentAppType { + case bean.ArgoCd: + triggerEvent.PerformChartPush = true + triggerEvent.PerformDeploymentOnCluster = true + triggerEvent.GetManifestInResponse = false + triggerEvent.DeploymentAppType = bean.ArgoCd + triggerEvent.ManifestStorageType = bean2.ManifestStorageGit + case bean.Helm: + triggerEvent.PerformChartPush = false + triggerEvent.PerformDeploymentOnCluster = true + triggerEvent.GetManifestInResponse = false + triggerEvent.DeploymentAppType = bean.Helm + } + return triggerEvent +} + +func ValidateTriggerEvent(triggerEvent bean.TriggerEvent) (bool, error) { + switch triggerEvent.DeploymentAppType { + case bean.ArgoCd: + if !triggerEvent.PerformChartPush { + return false, errors2.New("For deployment type ArgoCd, PerformChartPush flag expected value = true, got false") + } + case bean.Helm: + return true, nil + case bean.GitOpsWithoutDeployment: + if triggerEvent.PerformDeploymentOnCluster { + return false, errors2.New("For deployment type GitOpsWithoutDeployment, PerformDeploymentOnCluster flag expected value = false, got value = true") + } + case bean.ManifestDownload: + if triggerEvent.PerformChartPush { + return false, errors3.New("For deployment type ManifestDownload, PerformChartPush flag expected value = false, got true") + } + if triggerEvent.PerformDeploymentOnCluster { + return false, errors3.New("For deployment type ManifestDownload, PerformDeploymentOnCluster flag expected value = false, got true") + } + } + return true, nil +} diff --git a/pkg/deployment/trigger/devtronApps/wire_devtronAppsDeployTrigger.go b/pkg/deployment/trigger/devtronApps/wire_devtronAppsDeployTrigger.go new file mode 100644 index 0000000000..aed7b38f80 --- /dev/null +++ b/pkg/deployment/trigger/devtronApps/wire_devtronAppsDeployTrigger.go @@ -0,0 +1,8 @@ +package devtronApps + +import "github.com/google/wire" + +var DevtronAppsDeployTriggerWireSet = wire.NewSet( + NewTriggerServiceImpl, + wire.Bind(new(TriggerService), new(*TriggerServiceImpl)), +) diff --git a/pkg/deployment/wire_deployment.go b/pkg/deployment/wire_deployment.go index 5a959cd701..610b6a8ea4 100644 --- a/pkg/deployment/wire_deployment.go +++ b/pkg/deployment/wire_deployment.go @@ -3,6 +3,7 @@ package deployment import ( "github.com/devtron-labs/devtron/pkg/deployment/gitOps" "github.com/devtron-labs/devtron/pkg/deployment/manifest" + "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps" "github.com/google/wire" ) @@ -11,4 +12,5 @@ import ( var DeploymentWireSet = wire.NewSet( gitOps.GitOpsWireSet, manifest.DeploymentManifestWireSet, + devtronApps.DevtronAppsDeployTriggerWireSet, ) diff --git a/pkg/eventProcessor/CentralEventProcessorService.go b/pkg/eventProcessor/CentralEventProcessorService.go new file mode 100644 index 0000000000..a7d36477c0 --- /dev/null +++ b/pkg/eventProcessor/CentralEventProcessorService.go @@ -0,0 +1,44 @@ +package eventProcessor + +import ( + "github.com/devtron-labs/devtron/pkg/eventProcessor/in" + "go.uber.org/zap" +) + +type CentralEventProcessor struct { + logger *zap.SugaredLogger + workflowEventProcessor *in.WorkflowEventProcessorImpl +} + +func NewCentralEventProcessor(workflowEventProcessor *in.WorkflowEventProcessorImpl, + logger *zap.SugaredLogger) (*CentralEventProcessor, error) { + cep := &CentralEventProcessor{ + workflowEventProcessor: workflowEventProcessor, + logger: logger, + } + err := cep.SubscribeAll() + if err != nil { + return nil, err + } + return cep, nil +} + +func (impl *CentralEventProcessor) SubscribeAll() error { + var err error + err = impl.workflowEventProcessor.SubscribeCdStageCompleteEvent() + if err != nil { + impl.logger.Errorw("error, SubscribeCdStageCompleteEvent", "err", err) + return err + } + err = impl.workflowEventProcessor.SubscribeTriggerBulkAction() + if err != nil { + impl.logger.Errorw("error, SubscribeTriggerBulkAction", "err", err) + return err + } + err = impl.workflowEventProcessor.SubscribeHibernateBulkAction() + if err != nil { + impl.logger.Errorw("error, SubscribeHibernateBulkAction", "err", err) + return err + } + return nil +} diff --git a/pkg/eventProcessor/bean/workflowEventBean.go b/pkg/eventProcessor/bean/workflowEventBean.go new file mode 100644 index 0000000000..831cea2f2b --- /dev/null +++ b/pkg/eventProcessor/bean/workflowEventBean.go @@ -0,0 +1,27 @@ +package bean + +import ( + "github.com/devtron-labs/devtron/api/bean" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "time" +) + +type CdStageCompleteEvent struct { + CiProjectDetails []bean3.CiProjectDetails `json:"ciProjectDetails"` + WorkflowId int `json:"workflowId"` + WorkflowRunnerId int `json:"workflowRunnerId"` + CdPipelineId int `json:"cdPipelineId"` + TriggeredBy int32 `json:"triggeredBy"` + StageYaml string `json:"stageYaml"` + ArtifactLocation string `json:"artifactLocation"` + PipelineName string `json:"pipelineName"` + CiArtifactDTO pipelineConfig.CiArtifactDTO `json:"ciArtifactDTO"` + PluginRegistryArtifactDetails map[string][]string `json:"PluginRegistryArtifactDetails"` +} + +type AsyncCdDeployEvent struct { + ValuesOverrideRequest *bean.ValuesOverrideRequest `json:"valuesOverrideRequest"` + TriggeredAt time.Time `json:"triggeredAt"` + TriggeredBy int32 `json:"triggeredBy"` +} diff --git a/pkg/eventProcessor/in/WorkflowEventProcessorService.go b/pkg/eventProcessor/in/WorkflowEventProcessorService.go new file mode 100644 index 0000000000..ee1576c886 --- /dev/null +++ b/pkg/eventProcessor/in/WorkflowEventProcessorService.go @@ -0,0 +1,249 @@ +package in + +import ( + "encoding/json" + pubsub "github.com/devtron-labs/common-lib/pubsub-lib" + "github.com/devtron-labs/common-lib/pubsub-lib/model" + bean2 "github.com/devtron-labs/devtron/api/bean" + "github.com/devtron-labs/devtron/internal/sql/repository" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + bean4 "github.com/devtron-labs/devtron/pkg/auth/user/bean" + "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" + "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/workflow/cd" + "github.com/devtron-labs/devtron/pkg/workflow/cd/adapter" + bean3 "github.com/devtron-labs/devtron/pkg/workflow/cd/bean" + "github.com/devtron-labs/devtron/util/argo" + "go.uber.org/zap" + "k8s.io/utils/pointer" + "strconv" +) + +type WorkflowEventProcessorImpl struct { + logger *zap.SugaredLogger + pubSubClient *pubsub.PubSubClientServiceImpl + cdWorkflowService cd.CdWorkflowService + cdWorkflowRunnerService cd.CdWorkflowRunnerService + workflowDagExecutor pipeline.WorkflowDagExecutor + argoUserService argo.ArgoUserService + //repositories import, to be removed + pipelineRepository pipelineConfig.PipelineRepository + ciArtifactRepository repository.CiArtifactRepository +} + +func NewWorkflowEventProcessorImpl(logger *zap.SugaredLogger, + pubSubClient *pubsub.PubSubClientServiceImpl, + cdWorkflowService cd.CdWorkflowService, + cdWorkflowRunnerService cd.CdWorkflowRunnerService, + workflowDagExecutor pipeline.WorkflowDagExecutor, + argoUserService argo.ArgoUserService, + pipelineRepository pipelineConfig.PipelineRepository, + ciArtifactRepository repository.CiArtifactRepository) (*WorkflowEventProcessorImpl, error) { + impl := &WorkflowEventProcessorImpl{ + logger: logger, + pubSubClient: pubSubClient, + cdWorkflowService: cdWorkflowService, + cdWorkflowRunnerService: cdWorkflowRunnerService, + argoUserService: argoUserService, + workflowDagExecutor: workflowDagExecutor, + pipelineRepository: pipelineRepository, + ciArtifactRepository: ciArtifactRepository, + } + return impl, nil +} + +func (impl *WorkflowEventProcessorImpl) SubscribeCdStageCompleteEvent() error { + callback := func(msg *model.PubSubMsg) { + cdStageCompleteEvent := bean.CdStageCompleteEvent{} + err := json.Unmarshal([]byte(msg.Data), &cdStageCompleteEvent) + if err != nil { + impl.logger.Errorw("error while unmarshalling cdStageCompleteEvent object", "err", err, "msg", msg.Data) + return + } + wfr, err := impl.cdWorkflowRunnerService.FindWorkflowRunnerById(cdStageCompleteEvent.WorkflowRunnerId) + if err != nil { + impl.logger.Errorw("could not get wf runner", "err", err) + return + } + triggerContext := pipeline.TriggerContext{ + ReferenceId: pointer.String(msg.MsgId), + } + if wfr.WorkflowType == bean2.CD_WORKFLOW_TYPE_PRE { + impl.logger.Debugw("received pre stage success event for workflow runner ", "wfId", strconv.Itoa(wfr.Id)) + err = impl.workflowDagExecutor.HandlePreStageSuccessEvent(triggerContext, cdStageCompleteEvent) + if err != nil { + impl.logger.Errorw("deployment success event error", "err", err) + return + } + } else if wfr.WorkflowType == bean2.CD_WORKFLOW_TYPE_POST { + impl.logger.Debugw("received post stage success event for workflow runner ", "wfId", strconv.Itoa(wfr.Id)) + err = impl.workflowDagExecutor.HandlePostStageSuccessEvent(triggerContext, wfr.CdWorkflowId, cdStageCompleteEvent.CdPipelineId, cdStageCompleteEvent.TriggeredBy, cdStageCompleteEvent.PluginRegistryArtifactDetails) + if err != nil { + impl.logger.Errorw("deployment success event error", "err", err) + return + } + } + } + + // add required logging here + var loggerFunc pubsub.LoggerFunc = func(msg model.PubSubMsg) (string, []interface{}) { + cdStageCompleteEvent := bean.CdStageCompleteEvent{} + err := json.Unmarshal([]byte(msg.Data), &cdStageCompleteEvent) + if err != nil { + return "error while unmarshalling cdStageCompleteEvent object", []interface{}{"err", err, "msg", msg.Data} + } + return "got message for cd stage completion", []interface{}{"workflowRunnerId", cdStageCompleteEvent.WorkflowRunnerId, "workflowId", cdStageCompleteEvent.WorkflowId, "cdPipelineId", cdStageCompleteEvent.CdPipelineId} + } + + validations := impl.workflowDagExecutor.GetTriggerValidateFuncs() + + err := impl.pubSubClient.Subscribe(pubsub.CD_STAGE_COMPLETE_TOPIC, callback, loggerFunc, validations...) + if err != nil { + impl.logger.Error("error", "err", err) + return err + } + return nil +} + +func (impl *WorkflowEventProcessorImpl) SubscribeTriggerBulkAction() error { + callback := func(msg *model.PubSubMsg) { + cdWorkflow := new(pipelineConfig.CdWorkflow) + err := json.Unmarshal([]byte(msg.Data), cdWorkflow) + if err != nil { + impl.logger.Error("Error while unmarshalling cdWorkflow json object", "error", err) + return + } + wf := &bean3.CdWorkflowDto{ + Id: cdWorkflow.Id, + CiArtifactId: cdWorkflow.CiArtifactId, + PipelineId: cdWorkflow.PipelineId, + UserId: bean4.SYSTEM_USER_ID, + } + latest, err := impl.cdWorkflowService.CheckIfLatestWf(cdWorkflow.PipelineId, cdWorkflow.Id) + if err != nil { + impl.logger.Errorw("error in determining latest", "wf", cdWorkflow, "err", err) + wf.WorkflowStatus = pipelineConfig.DEQUE_ERROR + err = impl.cdWorkflowService.UpdateWorkFlow(wf) + if err != nil { + impl.logger.Errorw("error in updating wf", "err", err, "req", wf) + } + return + } + if !latest { + wf.WorkflowStatus = pipelineConfig.DROPPED_STALE + err = impl.cdWorkflowService.UpdateWorkFlow(wf) + if err != nil { + impl.logger.Errorw("error in updating wf", "err", err, "req", wf) + } + return + } + pipelineObj, err := impl.pipelineRepository.FindById(cdWorkflow.PipelineId) + if err != nil { + impl.logger.Errorw("error in fetching pipeline", "err", err) + wf.WorkflowStatus = pipelineConfig.TRIGGER_ERROR + err = impl.cdWorkflowService.UpdateWorkFlow(wf) + if err != nil { + impl.logger.Errorw("error in updating wf", "err", err, "req", wf) + } + return + } + artifact, err := impl.ciArtifactRepository.Get(cdWorkflow.CiArtifactId) + if err != nil { + impl.logger.Errorw("error in fetching artefact", "err", err) + wf.WorkflowStatus = pipelineConfig.TRIGGER_ERROR + err = impl.cdWorkflowService.UpdateWorkFlow(wf) + if err != nil { + impl.logger.Errorw("error in updating wf", "err", err, "req", wf) + } + return + } + // Migration of deprecated DataSource Type + if artifact.IsMigrationRequired() { + migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(artifact.Id) + if migrationErr != nil { + impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", artifact.Id) + } + } + triggerContext := pipeline.TriggerContext{ + ReferenceId: pointer.String(msg.MsgId), + } + + triggerRequest := pipeline.TriggerRequest{ + CdWf: adapter.ConvertCdWorkflowDtoToDbObj(wf), //TODO: update object from db to dto + Artifact: artifact, + Pipeline: pipelineObj, + TriggeredBy: cdWorkflow.CreatedBy, + ApplyAuth: false, + TriggerContext: triggerContext, + } + err = impl.workflowDagExecutor.TriggerStageForBulk(triggerRequest) + if err != nil { + impl.logger.Errorw("error in cd trigger ", "err", err) + wf.WorkflowStatus = pipelineConfig.TRIGGER_ERROR + } else { + wf.WorkflowStatus = pipelineConfig.WF_STARTED + } + err = impl.cdWorkflowService.UpdateWorkFlow(wf) + if err != nil { + impl.logger.Errorw("error in updating wf", "err", err, "req", wf) + } + } + + // add required logging here + var loggerFunc pubsub.LoggerFunc = func(msg model.PubSubMsg) (string, []interface{}) { + cdWorkflow := new(pipelineConfig.CdWorkflow) + err := json.Unmarshal([]byte(msg.Data), cdWorkflow) + if err != nil { + return "error while unmarshalling cdWorkflow json object", []interface{}{"error", err} + } + return "got message for bulk deploy", []interface{}{"cdWorkflowId", cdWorkflow.Id} + } + + validations := impl.workflowDagExecutor.GetTriggerValidateFuncs() + return impl.pubSubClient.Subscribe(pubsub.BULK_DEPLOY_TOPIC, callback, loggerFunc, validations...) +} + +func (impl *WorkflowEventProcessorImpl) SubscribeHibernateBulkAction() error { + callback := func(msg *model.PubSubMsg) { + deploymentGroupAppWithEnv := new(pipeline.DeploymentGroupAppWithEnv) + err := json.Unmarshal([]byte(msg.Data), deploymentGroupAppWithEnv) + if err != nil { + impl.logger.Error("Error while unmarshalling deploymentGroupAppWithEnv json object", err) + return + } + + stopAppRequest := &pipeline.StopAppRequest{ + AppId: deploymentGroupAppWithEnv.AppId, + EnvironmentId: deploymentGroupAppWithEnv.EnvironmentId, + UserId: deploymentGroupAppWithEnv.UserId, + RequestType: deploymentGroupAppWithEnv.RequestType, + } + ctx, err := impl.argoUserService.BuildACDContext() + if err != nil { + impl.logger.Errorw("error in creating acd sync context", "err", err) + return + } + triggerContext := pipeline.TriggerContext{ + ReferenceId: pointer.String(msg.MsgId), + Context: ctx, + } + _, err = impl.workflowDagExecutor.StopStartApp(triggerContext, stopAppRequest) + if err != nil { + impl.logger.Errorw("error in stop app request", "err", err) + return + } + } + + // add required logging here + var loggerFunc pubsub.LoggerFunc = func(msg model.PubSubMsg) (string, []interface{}) { + deploymentGroupAppWithEnv := new(pipeline.DeploymentGroupAppWithEnv) + err := json.Unmarshal([]byte(msg.Data), deploymentGroupAppWithEnv) + if err != nil { + return "error while unmarshalling deploymentGroupAppWithEnv json object", []interface{}{"err", err} + } + return "got message for bulk hibernate", []interface{}{"deploymentGroupId", deploymentGroupAppWithEnv.DeploymentGroupId, "appId", deploymentGroupAppWithEnv.AppId, "environmentId", deploymentGroupAppWithEnv.EnvironmentId} + } + + err := impl.pubSubClient.Subscribe(pubsub.BULK_HIBERNATE_TOPIC, callback, loggerFunc) + return err +} diff --git a/pkg/eventProcessor/in/wire_eventProcessorIn.go b/pkg/eventProcessor/in/wire_eventProcessorIn.go new file mode 100644 index 0000000000..1d0b2083ec --- /dev/null +++ b/pkg/eventProcessor/in/wire_eventProcessorIn.go @@ -0,0 +1,7 @@ +package in + +import "github.com/google/wire" + +var EventProcessorInWireSet = wire.NewSet( + NewWorkflowEventProcessorImpl, +) diff --git a/pkg/eventProcessor/wire_eventProcessor.go b/pkg/eventProcessor/wire_eventProcessor.go new file mode 100644 index 0000000000..8ed898fb1e --- /dev/null +++ b/pkg/eventProcessor/wire_eventProcessor.go @@ -0,0 +1,11 @@ +package eventProcessor + +import ( + "github.com/devtron-labs/devtron/pkg/eventProcessor/in" + "github.com/google/wire" +) + +var EventProcessorWireSet = wire.NewSet( + NewCentralEventProcessor, + in.EventProcessorInWireSet, +) diff --git a/pkg/pipeline/AppDeploymentTypeChangeManager.go b/pkg/pipeline/AppDeploymentTypeChangeManager.go index fcc6e1563c..cc2928944e 100644 --- a/pkg/pipeline/AppDeploymentTypeChangeManager.go +++ b/pkg/pipeline/AppDeploymentTypeChangeManager.go @@ -31,6 +31,7 @@ import ( "github.com/devtron-labs/devtron/pkg/bean" chartService "github.com/devtron-labs/devtron/pkg/chart" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" + bean3 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/juju/errors" "go.uber.org/zap" "strconv" @@ -53,7 +54,7 @@ type AppDeploymentTypeChangeManager interface { //DeleteDeploymentAppsForEnvironment : takes in environment id and current deployment app type // and deletes all the cd pipelines for that deployment type in all apps that belongs to // that environment. - DeleteDeploymentAppsForEnvironment(ctx context.Context, environmentId int, currentDeploymentAppType bean.DeploymentType, exclusionList []int, includeApps []int, userId int32) (*bean.DeploymentAppTypeChangeResponse, error) + DeleteDeploymentAppsForEnvironment(ctx context.Context, environmentId int, currentDeploymentAppType bean3.DeploymentType, exclusionList []int, includeApps []int, userId int32) (*bean.DeploymentAppTypeChangeResponse, error) } type AppDeploymentTypeChangeManagerImpl struct { @@ -102,13 +103,13 @@ func (impl *AppDeploymentTypeChangeManagerImpl) ChangeDeploymentType(ctx context request *bean.DeploymentAppTypeChangeRequest) (*bean.DeploymentAppTypeChangeResponse, error) { var response *bean.DeploymentAppTypeChangeResponse - var deleteDeploymentType bean.DeploymentType + var deleteDeploymentType bean3.DeploymentType var err error - if request.DesiredDeploymentType == bean.ArgoCd { - deleteDeploymentType = bean.Helm + if request.DesiredDeploymentType == bean3.ArgoCd { + deleteDeploymentType = bean3.Helm } else { - deleteDeploymentType = bean.ArgoCd + deleteDeploymentType = bean3.ArgoCd } // Force delete apps @@ -222,12 +223,12 @@ func (impl *AppDeploymentTypeChangeManagerImpl) ChangePipelineDeploymentType(ctx TriggeredPipelines: make([]*bean.CdPipelineTrigger, 0), } - var deleteDeploymentType bean.DeploymentType + var deleteDeploymentType bean3.DeploymentType - if request.DesiredDeploymentType == bean.ArgoCd { - deleteDeploymentType = bean.Helm + if request.DesiredDeploymentType == bean3.ArgoCd { + deleteDeploymentType = bean3.Helm } else { - deleteDeploymentType = bean.ArgoCd + deleteDeploymentType = bean3.ArgoCd } pipelines, err := impl.pipelineRepository.FindActiveByEnvIdAndDeploymentType(request.EnvId, @@ -425,7 +426,7 @@ func (impl *AppDeploymentTypeChangeManagerImpl) DeleteDeploymentApps(ctx context var err error // delete request - if pipeline.DeploymentAppType == bean.ArgoCd { + if pipeline.DeploymentAppType == bean3.ArgoCd { err = impl.deleteArgoCdApp(ctx, pipeline, deploymentAppName, true) } else { @@ -507,7 +508,7 @@ func (impl *AppDeploymentTypeChangeManagerImpl) DeleteDeploymentApps(ctx context } func (impl *AppDeploymentTypeChangeManagerImpl) DeleteDeploymentAppsForEnvironment(ctx context.Context, environmentId int, - currentDeploymentAppType bean.DeploymentType, exclusionList []int, includeApps []int, userId int32) (*bean.DeploymentAppTypeChangeResponse, error) { + currentDeploymentAppType bean3.DeploymentType, exclusionList []int, includeApps []int, userId int32) (*bean.DeploymentAppTypeChangeResponse, error) { // fetch active pipelines from database for the given environment id and current deployment app type pipelines, err := impl.pipelineRepository.FindActiveByEnvIdAndDeploymentType(environmentId, @@ -548,7 +549,7 @@ func (impl *AppDeploymentTypeChangeManagerImpl) isPipelineInfoValid(pipeline *pi func (impl *AppDeploymentTypeChangeManagerImpl) handleNotHealthyAppsIfArgoDeploymentType(pipeline *pipelineConfig.Pipeline, failedPipelines []*bean.DeploymentChangeStatus) ([]*bean.DeploymentChangeStatus, error) { - if pipeline.DeploymentAppType == bean.ArgoCd { + if pipeline.DeploymentAppType == bean3.ArgoCd { // check if app status is Healthy status, err := impl.appStatusRepository.Get(pipeline.AppId, pipeline.EnvironmentId) @@ -579,7 +580,7 @@ func (impl *AppDeploymentTypeChangeManagerImpl) handleNotHealthyAppsIfArgoDeploy func (impl *AppDeploymentTypeChangeManagerImpl) handleNotDeployedAppsIfArgoDeploymentType(pipeline *pipelineConfig.Pipeline, failedPipelines []*bean.DeploymentChangeStatus) ([]*bean.DeploymentChangeStatus, error) { - if pipeline.DeploymentAppType == string(bean.ArgoCd) { + if pipeline.DeploymentAppType == string(bean3.ArgoCd) { // check if app status is Healthy status, err := impl.appStatusRepository.Get(pipeline.AppId, pipeline.EnvironmentId) @@ -627,7 +628,7 @@ func (impl *AppDeploymentTypeChangeManagerImpl) FetchDeletedApp(ctx context.Cont deploymentAppName := fmt.Sprintf("%s-%s", pipeline.App.AppName, pipeline.Environment.Name) var err error - if pipeline.DeploymentAppType == string(bean.ArgoCd) { + if pipeline.DeploymentAppType == bean3.ArgoCd { appIdentifier := &service.AppIdentifier{ ClusterId: pipeline.Environment.ClusterId, ReleaseName: pipeline.DeploymentAppName, diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index 0e4ea796fc..b1e15cf2f8 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -575,7 +575,7 @@ func (impl *CdHandlerImpl) CheckHelmAppStatusPeriodicallyAndUpdateInDb(helmPipel wfr.UpdatedBy = 1 wfr.UpdatedOn = time.Now() if wfr.Status == pipelineConfig.WorkflowFailed { - err = impl.workflowDagExecutor.MarkPipelineStatusTimelineFailed(wfr, errors.New(pipelineConfig.NEW_DEPLOYMENT_INITIATED)) + err = impl.pipelineStatusTimelineService.MarkPipelineStatusTimelineFailed(wfr.RefCdWorkflowRunnerId, pipelineConfig.NEW_DEPLOYMENT_INITIATED) if err != nil { impl.Logger.Errorw("error updating CdPipelineStatusTimeline", "err", err) return err @@ -587,7 +587,7 @@ func (impl *CdHandlerImpl) CheckHelmAppStatusPeriodicallyAndUpdateInDb(helmPipel return err } if slices.Contains(pipelineConfig.WfrTerminalStatusList, wfr.Status) { - impl.workflowDagExecutor.UpdateTriggerCDMetricsOnFinish(wfr) + util3.TriggerCDMetrics(pipelineConfig.GetTriggerMetricsFromRunnerObj(wfr), impl.config.ExposeCDMetrics) } impl.Logger.Infow("updated workflow runner status for helm app", "wfr", wfr) diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index c62ed46d83..d053cf4539 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -20,24 +20,22 @@ package pipeline import ( "context" "encoding/json" - errors3 "errors" "fmt" bean6 "github.com/devtron-labs/devtron/api/helm-app/bean" "github.com/devtron-labs/devtron/api/helm-app/gRPC" client2 "github.com/devtron-labs/devtron/api/helm-app/service" - "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" - "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" - bean5 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/bean" - "path" + "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps" + "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/helper" + bean7 "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" + "github.com/devtron-labs/devtron/pkg/workflow/cd" "strconv" "strings" "sync" "time" application3 "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" - "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" "github.com/aws/aws-sdk-go/service/autoscaling" blob_storage "github.com/devtron-labs/common-lib/blob-storage" "github.com/devtron-labs/common-lib/pubsub-lib/model" @@ -48,7 +46,6 @@ import ( application2 "github.com/devtron-labs/devtron/client/argocdServer/application" gitSensorClient "github.com/devtron-labs/devtron/client/gitSensor" "github.com/devtron-labs/devtron/internal/middleware" - bean4 "github.com/devtron-labs/devtron/pkg/app/bean" "github.com/devtron-labs/devtron/pkg/app/status" "github.com/devtron-labs/devtron/pkg/auth/user" "github.com/devtron-labs/devtron/pkg/chartRepo/repository" @@ -71,11 +68,7 @@ import ( "github.com/tidwall/gjson" "github.com/tidwall/sjson" "go.opentelemetry.io/otel" - "google.golang.org/grpc/codes" - status2 "google.golang.org/grpc/status" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/helm/pkg/proto/hapi/chart" - "k8s.io/utils/pointer" "k8s.io/utils/strings/slices" "github.com/devtron-labs/devtron/internal/sql/repository/appWorkflow" @@ -105,23 +98,24 @@ import ( type WorkflowDagExecutor interface { HandleCiSuccessEvent(triggerContext TriggerContext, artifact *repository.CiArtifact, async bool, triggeredBy int32) error HandleWebhookExternalCiEvent(artifact *repository.CiArtifact, triggeredBy int32, externalCiId int, auth func(token string, projectObject string, envObject string) bool, token string) (bool, error) - HandlePreStageSuccessEvent(triggerContext TriggerContext, cdStageCompleteEvent CdStageCompleteEvent) error + HandlePreStageSuccessEvent(triggerContext TriggerContext, cdStageCompleteEvent bean7.CdStageCompleteEvent) error HandleDeploymentSuccessEvent(triggerContext TriggerContext, pipelineOverride *chartConfig.PipelineOverride) error HandlePostStageSuccessEvent(triggerContext TriggerContext, cdWorkflowId int, cdPipelineId int, triggeredBy int32, pluginRegistryImageDetails map[string][]string) error - Subscribe() error + TriggerPostStage(request TriggerRequest) error TriggerPreStage(request TriggerRequest) error TriggerDeployment(request TriggerRequest) error ManualCdTrigger(triggerContext TriggerContext, overrideRequest *bean.ValuesOverrideRequest) (int, error) TriggerBulkDeploymentAsync(requests []*BulkTriggerRequest, UserId int32) (interface{}, error) - StopStartApp(triggerContext TriggerContext, stopRequest *StopAppRequest) (int, error) + TriggerStageForBulk(triggerRequest TriggerRequest) error + TriggerBulkHibernateAsync(request StopDeploymentGroupRequest, ctx context.Context) (interface{}, error) + StopStartApp(triggerContext TriggerContext, stopRequest *StopAppRequest) (int, error) RotatePods(ctx context.Context, podRotateRequest *PodRotateRequest) (*k8s.RotatePodResponse, error) + MarkCurrentDeploymentFailed(runner *pipelineConfig.CdWorkflowRunner, releaseErr error, triggeredBy int32) error UpdateWorkflowRunnerStatusForDeployment(appIdentifier *client2.AppIdentifier, wfr *pipelineConfig.CdWorkflowRunner, skipReleaseNotFound bool) bool OnDeleteCdPipelineEvent(pipelineId int, triggeredBy int32) - MarkPipelineStatusTimelineFailed(runner *pipelineConfig.CdWorkflowRunner, releaseErr error) error - UpdateTriggerCDMetricsOnFinish(runner *pipelineConfig.CdWorkflowRunner) GetTriggerValidateFuncs() []pubsub.ValidateMsg } @@ -187,18 +181,19 @@ type WorkflowDagExecutorImpl struct { configMapHistoryRepository repository3.ConfigMapHistoryRepository helmAppService client2.HelmAppService //TODO fix me next - helmAppClient gRPC.HelmAppClient //TODO refactoring: use helm app service instead - environmentConfigRepository chartConfig.EnvConfigOverrideRepository - mergeUtil *util.MergeUtil - acdClient application2.ServiceClient - argoClientWrapperService argocdServer.ArgoClientWrapperService - customTagService CustomTagService - ACDConfig *argocdServer.ACDConfig - deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService - chartRefService chartRef.ChartRefService - gitOpsConfigReadService config.GitOpsConfigReadService - gitOperationService git.GitOperationService - imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService + helmAppClient gRPC.HelmAppClient //TODO refactoring: use helm app service instead + environmentConfigRepository chartConfig.EnvConfigOverrideRepository + mergeUtil *util.MergeUtil + acdClient application2.ServiceClient + argoClientWrapperService argocdServer.ArgoClientWrapperService + customTagService CustomTagService + ACDConfig *argocdServer.ACDConfig + deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService + chartRefService chartRef.ChartRefService + imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService + + cdWorkflowCommonService cd.CdWorkflowCommonService + cdTriggerService devtronApps.TriggerService } const kedaAutoscaling = "kedaAutoscaling" @@ -220,7 +215,6 @@ const ( CHILD_CD_CLUSTER_NAME_PREFIX = "CHILD_CD_CLUSTER_NAME" CHILD_CD_COUNT = "CHILD_CD_COUNT" DEVTRON_SYSTEM_USER_ID = 1 - ARGOCD_SYNC_ERROR = "error in syncing argoCD app" ARGOCD_REFRESH_ERROR = "Error in refreshing argocd app" ) @@ -229,19 +223,6 @@ type DevtronAppReleaseContextType struct { RunnerId int } -type CdStageCompleteEvent struct { - CiProjectDetails []bean3.CiProjectDetails `json:"ciProjectDetails"` - WorkflowId int `json:"workflowId"` - WorkflowRunnerId int `json:"workflowRunnerId"` - CdPipelineId int `json:"cdPipelineId"` - TriggeredBy int32 `json:"triggeredBy"` - StageYaml string `json:"stageYaml"` - ArtifactLocation string `json:"artifactLocation"` - PipelineName string `json:"pipelineName"` - CiArtifactDTO pipelineConfig.CiArtifactDTO `json:"ciArtifactDTO"` - PluginRegistryArtifactDetails map[string][]string `json:"PluginRegistryArtifactDetails"` -} - type TriggerRequest struct { CdWf *pipelineConfig.CdWorkflow Pipeline *pipelineConfig.Pipeline @@ -318,10 +299,9 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi ACDConfig *argocdServer.ACDConfig, deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService, chartRefService chartRef.ChartRefService, - gitOpsConfigReadService config.GitOpsConfigReadService, - gitOperationService git.GitOperationService, imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService, -) *WorkflowDagExecutorImpl { + cdWorkflowCommonService cd.CdWorkflowCommonService, + cdTriggerService devtronApps.TriggerService) *WorkflowDagExecutorImpl { wde := &WorkflowDagExecutorImpl{logger: Logger, pipelineRepository: pipelineRepository, cdWorkflowRepository: cdWorkflowRepository, @@ -388,9 +368,9 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi ACDConfig: ACDConfig, deployedAppMetricsService: deployedAppMetricsService, chartRefService: chartRefService, - gitOpsConfigReadService: gitOpsConfigReadService, - gitOperationService: gitOperationService, imageDigestPolicyService: imageDigestPolicyService, + cdWorkflowCommonService: cdWorkflowCommonService, + cdTriggerService: cdTriggerService, } config, err := types.GetCdConfig() if err != nil { @@ -402,18 +382,6 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi return nil } wde.appServiceConfig = appServiceConfig - err = wde.Subscribe() - if err != nil { - return nil - } - err = wde.subscribeTriggerBulkAction() - if err != nil { - return nil - } - err = wde.subscribeHibernateBulkAction() - if err != nil { - return nil - } err = wde.SubscribeDevtronAsyncHelmInstallRequest() if err != nil { return nil @@ -422,61 +390,8 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi return wde } -func (impl *WorkflowDagExecutorImpl) Subscribe() error { - callback := func(msg *model.PubSubMsg) { - cdStageCompleteEvent := CdStageCompleteEvent{} - err := json.Unmarshal([]byte(string(msg.Data)), &cdStageCompleteEvent) - if err != nil { - impl.logger.Errorw("error while unmarshalling cdStageCompleteEvent object", "err", err, "msg", string(msg.Data)) - return - } - wf, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(cdStageCompleteEvent.WorkflowRunnerId) - if err != nil { - impl.logger.Errorw("could not get wf runner", "err", err) - return - } - triggerContext := TriggerContext{ - ReferenceId: pointer.String(msg.MsgId), - } - if wf.WorkflowType == bean.CD_WORKFLOW_TYPE_PRE { - impl.logger.Debugw("received pre stage success event for workflow runner ", "wfId", strconv.Itoa(wf.Id)) - err = impl.HandlePreStageSuccessEvent(triggerContext, cdStageCompleteEvent) - if err != nil { - impl.logger.Errorw("deployment success event error", "err", err) - return - } - } else if wf.WorkflowType == bean.CD_WORKFLOW_TYPE_POST { - impl.logger.Debugw("received post stage success event for workflow runner ", "wfId", strconv.Itoa(wf.Id)) - err = impl.HandlePostStageSuccessEvent(triggerContext, wf.CdWorkflowId, cdStageCompleteEvent.CdPipelineId, cdStageCompleteEvent.TriggeredBy, cdStageCompleteEvent.PluginRegistryArtifactDetails) - if err != nil { - impl.logger.Errorw("deployment success event error", "err", err) - return - } - } - } - - // add required logging here - var loggerFunc pubsub.LoggerFunc = func(msg model.PubSubMsg) (string, []interface{}) { - cdStageCompleteEvent := CdStageCompleteEvent{} - err := json.Unmarshal([]byte(string(msg.Data)), &cdStageCompleteEvent) - if err != nil { - return "error while unmarshalling cdStageCompleteEvent object", []interface{}{"err", err, "msg", string(msg.Data)} - } - return "got message for cd stage completion", []interface{}{"workflowRunnerId", cdStageCompleteEvent.WorkflowRunnerId, "workflowId", cdStageCompleteEvent.WorkflowId, "cdPipelineId", cdStageCompleteEvent.CdPipelineId} - } - - validations := impl.GetTriggerValidateFuncs() - - err := impl.pubsubClient.Subscribe(pubsub.CD_STAGE_COMPLETE_TOPIC, callback, loggerFunc, validations...) - if err != nil { - impl.logger.Error("error", "err", err) - return err - } - return nil -} - -func (impl *WorkflowDagExecutorImpl) extractOverrideRequestFromCDAsyncInstallEvent(msg *model.PubSubMsg) (*bean.AsyncCdDeployEvent, *client2.AppIdentifier, error) { - CDAsyncInstallNatsMessage := &bean.AsyncCdDeployEvent{} +func (impl *WorkflowDagExecutorImpl) extractOverrideRequestFromCDAsyncInstallEvent(msg *model.PubSubMsg) (*bean7.AsyncCdDeployEvent, *client2.AppIdentifier, error) { + CDAsyncInstallNatsMessage := &bean7.AsyncCdDeployEvent{} err := json.Unmarshal([]byte(msg.Data), CDAsyncInstallNatsMessage) if err != nil { impl.logger.Errorw("error in unmarshalling CD async install request nats message", "err", err) @@ -682,7 +597,7 @@ func (impl *WorkflowDagExecutorImpl) cleanUpDevtronAsyncHelmInstallRequest(pipel impl.cleanUpDevtronAppReleaseContextMap(pipelineId, wfrId) } -func (impl *WorkflowDagExecutorImpl) processDevtronAsyncHelmInstallRequest(CDAsyncInstallNatsMessage *bean.AsyncCdDeployEvent, appIdentifier *client2.AppIdentifier) { +func (impl *WorkflowDagExecutorImpl) processDevtronAsyncHelmInstallRequest(CDAsyncInstallNatsMessage *bean7.AsyncCdDeployEvent, appIdentifier *client2.AppIdentifier) { overrideRequest := CDAsyncInstallNatsMessage.ValuesOverrideRequest cdWfr, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(overrideRequest.WfrId) if err != nil { @@ -724,7 +639,7 @@ func (impl *WorkflowDagExecutorImpl) processDevtronAsyncHelmInstallRequest(CDAsy impl.UpdateReleaseContextForPipeline(overrideRequest.PipelineId, cdWfr.Id, cancel) //update workflow runner status, used in app workflow view - err = impl.UpdateCDWorkflowRunnerStatus(ctx, overrideRequest, CDAsyncInstallNatsMessage.TriggeredAt, pipelineConfig.WorkflowStarting, "") + err = impl.cdWorkflowCommonService.UpdateCDWorkflowRunnerStatus(ctx, overrideRequest, CDAsyncInstallNatsMessage.TriggeredAt, pipelineConfig.WorkflowStarting, "") if err != nil { impl.logger.Errorw("error in updating the workflow runner status, processDevtronAsyncHelmInstallRequest", "cdWfrId", cdWfr.Id, "err", err) return @@ -771,7 +686,7 @@ func (impl *WorkflowDagExecutorImpl) SubscribeDevtronAsyncHelmInstallRequest() e // add required logging here var loggerFunc pubsub.LoggerFunc = func(msg model.PubSubMsg) (string, []interface{}) { - CDAsyncInstallNatsMessage := &bean.AsyncCdDeployEvent{} + CDAsyncInstallNatsMessage := &bean7.AsyncCdDeployEvent{} err := json.Unmarshal([]byte(msg.Data), CDAsyncInstallNatsMessage) if err != nil { return "error in unmarshalling CD async install request nats message", []interface{}{"err", err} @@ -925,7 +840,7 @@ func (impl *WorkflowDagExecutorImpl) getPipelineStage(pipelineId int, stageType return stage, nil } -func (impl *WorkflowDagExecutorImpl) triggerStageForBulk(triggerRequest TriggerRequest, async bool) error { +func (impl *WorkflowDagExecutorImpl) TriggerStageForBulk(triggerRequest TriggerRequest) error { preStage, err := impl.getPipelineStage(triggerRequest.Pipeline.Id, repository4.PIPELINE_STAGE_TYPE_PRE_CD) if err != nil { @@ -954,7 +869,7 @@ func (impl *WorkflowDagExecutorImpl) triggerStageForBulk(triggerRequest TriggerR } } -func (impl *WorkflowDagExecutorImpl) HandlePreStageSuccessEvent(triggerContext TriggerContext, cdStageCompleteEvent CdStageCompleteEvent) error { +func (impl *WorkflowDagExecutorImpl) HandlePreStageSuccessEvent(triggerContext TriggerContext, cdStageCompleteEvent bean7.CdStageCompleteEvent) error { wfRunner, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(cdStageCompleteEvent.WorkflowRunnerId) if err != nil { return err @@ -2710,149 +2625,6 @@ func (impl *WorkflowDagExecutorImpl) triggerNatsEventForBulkAction(cdWorkflows [ } } -func (impl *WorkflowDagExecutorImpl) subscribeTriggerBulkAction() error { - callback := func(msg *model.PubSubMsg) { - cdWorkflow := new(pipelineConfig.CdWorkflow) - err := json.Unmarshal([]byte(string(msg.Data)), cdWorkflow) - if err != nil { - impl.logger.Error("Error while unmarshalling cdWorkflow json object", "error", err) - return - } - wf := &pipelineConfig.CdWorkflow{ - Id: cdWorkflow.Id, - CiArtifactId: cdWorkflow.CiArtifactId, - PipelineId: cdWorkflow.PipelineId, - AuditLog: sql.AuditLog{ - UpdatedOn: time.Now(), - }, - } - latest, err := impl.cdWorkflowRepository.IsLatestWf(cdWorkflow.PipelineId, cdWorkflow.Id) - if err != nil { - impl.logger.Errorw("error in determining latest", "wf", cdWorkflow, "err", err) - wf.WorkflowStatus = pipelineConfig.DEQUE_ERROR - impl.cdWorkflowRepository.UpdateWorkFlow(wf) - return - } - if !latest { - wf.WorkflowStatus = pipelineConfig.DROPPED_STALE - impl.cdWorkflowRepository.UpdateWorkFlow(wf) - return - } - pipeline, err := impl.pipelineRepository.FindById(cdWorkflow.PipelineId) - if err != nil { - impl.logger.Errorw("error in fetching pipeline", "err", err) - wf.WorkflowStatus = pipelineConfig.TRIGGER_ERROR - impl.cdWorkflowRepository.UpdateWorkFlow(wf) - return - } - artifact, err := impl.ciArtifactRepository.Get(cdWorkflow.CiArtifactId) - if err != nil { - impl.logger.Errorw("error in fetching artefact", "err", err) - wf.WorkflowStatus = pipelineConfig.TRIGGER_ERROR - impl.cdWorkflowRepository.UpdateWorkFlow(wf) - return - } - // Migration of deprecated DataSource Type - if artifact.IsMigrationRequired() { - migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(artifact.Id) - if migrationErr != nil { - impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", artifact.Id) - } - } - triggerContext := TriggerContext{ - ReferenceId: pointer.String(msg.MsgId), - } - - triggerRequest := TriggerRequest{ - CdWf: wf, - Artifact: artifact, - Pipeline: pipeline, - TriggeredBy: cdWorkflow.CreatedBy, - ApplyAuth: false, - TriggerContext: triggerContext, - } - err = impl.triggerStageForBulk(triggerRequest, false) - if err != nil { - impl.logger.Errorw("error in cd trigger ", "err", err) - wf.WorkflowStatus = pipelineConfig.TRIGGER_ERROR - } else { - wf.WorkflowStatus = pipelineConfig.WF_STARTED - } - impl.cdWorkflowRepository.UpdateWorkFlow(wf) - } - - // add required logging here - var loggerFunc pubsub.LoggerFunc = func(msg model.PubSubMsg) (string, []interface{}) { - cdWorkflow := new(pipelineConfig.CdWorkflow) - err := json.Unmarshal([]byte(string(msg.Data)), cdWorkflow) - if err != nil { - return "error while unmarshalling cdWorkflow json object", []interface{}{"error", err} - } - return "got message for bulk deploy", []interface{}{"cdWorkflowId", cdWorkflow.Id} - } - - validations := impl.GetTriggerValidateFuncs() - err := impl.pubsubClient.Subscribe(pubsub.BULK_DEPLOY_TOPIC, callback, loggerFunc, validations...) - return err -} - -func (impl *WorkflowDagExecutorImpl) subscribeHibernateBulkAction() error { - callback := func(msg *model.PubSubMsg) { - deploymentGroupAppWithEnv := new(DeploymentGroupAppWithEnv) - err := json.Unmarshal([]byte(string(msg.Data)), deploymentGroupAppWithEnv) - if err != nil { - impl.logger.Error("Error while unmarshalling deploymentGroupAppWithEnv json object", err) - return - } - - stopAppRequest := &StopAppRequest{ - AppId: deploymentGroupAppWithEnv.AppId, - EnvironmentId: deploymentGroupAppWithEnv.EnvironmentId, - UserId: deploymentGroupAppWithEnv.UserId, - RequestType: deploymentGroupAppWithEnv.RequestType, - } - ctx, err := impl.buildACDContext() - if err != nil { - impl.logger.Errorw("error in creating acd synch context", "err", err) - return - } - triggerContext := TriggerContext{ - ReferenceId: pointer.String(msg.MsgId), - Context: ctx, - } - _, err = impl.StopStartApp(triggerContext, stopAppRequest) - if err != nil { - impl.logger.Errorw("error in stop app request", "err", err) - return - } - } - - // add required logging here - var loggerFunc pubsub.LoggerFunc = func(msg model.PubSubMsg) (string, []interface{}) { - deploymentGroupAppWithEnv := new(DeploymentGroupAppWithEnv) - err := json.Unmarshal([]byte(string(msg.Data)), deploymentGroupAppWithEnv) - if err != nil { - return "error while unmarshalling deploymentGroupAppWithEnv json object", []interface{}{"err", err} - } - return "got message for bulk hibernate", []interface{}{"deploymentGroupId", deploymentGroupAppWithEnv.DeploymentGroupId, "appId", deploymentGroupAppWithEnv.AppId, "environmentId", deploymentGroupAppWithEnv.EnvironmentId} - } - - err := impl.pubsubClient.Subscribe(pubsub.BULK_HIBERNATE_TOPIC, callback, loggerFunc) - return err -} - -func (impl *WorkflowDagExecutorImpl) buildACDContext() (acdContext context.Context, err error) { - //this part only accessible for acd apps hibernation, if acd configured it will fetch latest acdToken, else it will return error - acdToken, err := impl.argoUserService.GetLatestDevtronArgoCdUserToken() - if err != nil { - impl.logger.Errorw("error in getting acd token", "err", err) - return nil, err - } - ctx := context.Background() - ctx = context.WithValue(ctx, "token", acdToken) - return ctx, nil -} - func extractTimelineFailedStatusDetails(err error) string { errorString := util.GetGRPCErrorDetailedMessage(err) switch errorString { @@ -2863,49 +2635,8 @@ func extractTimelineFailedStatusDetails(err error) string { } } -func (impl *WorkflowDagExecutorImpl) MarkPipelineStatusTimelineFailed(runner *pipelineConfig.CdWorkflowRunner, releaseErr error) error { - //creating cd pipeline status timeline for deployment failed - terminalStatusExists, timelineErr := impl.cdPipelineStatusTimelineRepo.CheckIfTerminalStatusTimelinePresentByWfrId(runner.Id) - if timelineErr != nil { - impl.logger.Errorw("error in checking if terminal status timeline exists by wfrId", "err", timelineErr, "wfrId", runner.Id) - return timelineErr - } - if !terminalStatusExists { - impl.logger.Infow("marking pipeline deployment failed", "err", releaseErr) - timeline := &pipelineConfig.PipelineStatusTimeline{ - CdWorkflowRunnerId: runner.Id, - Status: pipelineConfig.TIMELINE_STATUS_DEPLOYMENT_FAILED, - StatusDetail: extractTimelineFailedStatusDetails(releaseErr), - StatusTime: time.Now(), - AuditLog: sql.AuditLog{ - CreatedBy: 1, - CreatedOn: time.Now(), - UpdatedBy: 1, - UpdatedOn: time.Now(), - }, - } - timelineErr = impl.pipelineStatusTimelineService.SaveTimeline(timeline, nil, false) - if timelineErr != nil { - impl.logger.Errorw("error in creating timeline status for deployment fail", "err", timelineErr, "timeline", timeline) - } - } - return nil -} - -func (impl *WorkflowDagExecutorImpl) UpdateTriggerCDMetricsOnFinish(runner *pipelineConfig.CdWorkflowRunner) { - cdMetrics := util4.CDMetrics{ - AppName: runner.CdWorkflow.Pipeline.DeploymentAppName, - Status: runner.Status, - DeploymentType: runner.CdWorkflow.Pipeline.DeploymentAppType, - EnvironmentName: runner.CdWorkflow.Pipeline.Environment.Name, - Time: time.Since(runner.StartedOn).Seconds() - time.Since(runner.FinishedOn).Seconds(), - } - util4.TriggerCDMetrics(cdMetrics, impl.config.ExposeCDMetrics) - return -} - func (impl *WorkflowDagExecutorImpl) MarkCurrentDeploymentFailed(runner *pipelineConfig.CdWorkflowRunner, releaseErr error, triggeredBy int32) error { - err := impl.MarkPipelineStatusTimelineFailed(runner, releaseErr) + err := impl.pipelineStatusTimelineService.MarkPipelineStatusTimelineFailed(runner.Id, extractTimelineFailedStatusDetails(releaseErr)) if err != nil { impl.logger.Errorw("error updating CdPipelineStatusTimeline", "err", err, "releaseErr", releaseErr) return err @@ -2922,7 +2653,7 @@ func (impl *WorkflowDagExecutorImpl) MarkCurrentDeploymentFailed(runner *pipelin impl.logger.Errorw("error updating cd wf runner status", "err", releaseErr, "currentRunner", runner) return err1 } - impl.UpdateTriggerCDMetricsOnFinish(runner) + util4.TriggerCDMetrics(pipelineConfig.GetTriggerMetricsFromRunnerObj(runner), impl.config.ExposeCDMetrics) return nil } @@ -2963,7 +2694,7 @@ func (impl *WorkflowDagExecutorImpl) TriggerHelmAsyncRelease(overrideRequest *be return releaseNo, manifest, err } - event := &bean.AsyncCdDeployEvent{ + event := &bean7.AsyncCdDeployEvent{ ValuesOverrideRequest: overrideRequest, TriggeredAt: triggeredAt, TriggeredBy: triggeredBy, @@ -2979,7 +2710,7 @@ func (impl *WorkflowDagExecutorImpl) TriggerHelmAsyncRelease(overrideRequest *be if err != nil { impl.logger.Errorw("failed to publish trigger request event", "topic", pubsub.DEVTRON_CHART_INSTALL_TOPIC, "payload", payload, "err", err) //update workflow runner status, used in app workflow view - err1 = impl.UpdateCDWorkflowRunnerStatus(ctx, overrideRequest, triggeredAt, pipelineConfig.WorkflowFailed, err.Error()) + err1 = impl.cdWorkflowCommonService.UpdateCDWorkflowRunnerStatus(ctx, overrideRequest, triggeredAt, pipelineConfig.WorkflowFailed, err.Error()) if err1 != nil { impl.logger.Errorw("error in updating the workflow runner status, TriggerHelmAsyncRelease", "err", err1) } @@ -2987,7 +2718,7 @@ func (impl *WorkflowDagExecutorImpl) TriggerHelmAsyncRelease(overrideRequest *be } //update workflow runner status, used in app workflow view - err = impl.UpdateCDWorkflowRunnerStatus(ctx, overrideRequest, triggeredAt, pipelineConfig.WorkflowInQueue, "") + err = impl.cdWorkflowCommonService.UpdateCDWorkflowRunnerStatus(ctx, overrideRequest, triggeredAt, pipelineConfig.WorkflowInQueue, "") if err != nil { impl.logger.Errorw("error in updating the workflow runner status, TriggerHelmAsyncRelease", "err", err) return 0, manifest, err @@ -3006,8 +2737,8 @@ func (impl *WorkflowDagExecutorImpl) TriggerRelease(overrideRequest *bean.Values if overrideRequest.UserId == 0 { overrideRequest.UserId = triggeredBy } - triggerEvent := impl.GetTriggerEvent(overrideRequest.DeploymentAppType, triggeredAt, triggeredBy) - releaseNo, manifest, err = impl.TriggerPipeline(overrideRequest, valuesOverrideResponse, builtChartPath, triggerEvent, ctx) + triggerEvent := helper.GetTriggerEvent(overrideRequest.DeploymentAppType, triggeredAt, triggeredBy) + releaseNo, manifest, err = impl.cdTriggerService.TriggerPipeline(overrideRequest, valuesOverrideResponse, builtChartPath, triggerEvent, ctx) if err != nil { return 0, manifest, err } @@ -3061,7 +2792,7 @@ func (impl *WorkflowDagExecutorImpl) releasePipeline(pipeline *pipelineConfig.Pi } impl.SetPipelineFieldsInOverrideRequest(request, pipeline) - ctx, err := impl.buildACDContext() + ctx, err := impl.argoUserService.BuildACDContext() if err != nil { impl.logger.Errorw("error in creating acd sync context", "pipelineId", pipeline.Id, "artifactId", artifact.Id, "err", err) return err @@ -3088,113 +2819,7 @@ func (impl *WorkflowDagExecutorImpl) SetPipelineFieldsInOverrideRequest(override overrideRequest.DeploymentAppType = pipeline.DeploymentAppType } -func (impl *WorkflowDagExecutorImpl) GetTriggerEvent(deploymentAppType string, triggeredAt time.Time, deployedBy int32) bean.TriggerEvent { - // trigger event will decide whether to perform GitOps or deployment for a particular deployment app type - triggerEvent := bean.TriggerEvent{ - TriggeredBy: deployedBy, - TriggerdAt: triggeredAt, - } - switch deploymentAppType { - case bean2.ArgoCd: - triggerEvent.PerformChartPush = true - triggerEvent.PerformDeploymentOnCluster = true - triggerEvent.GetManifestInResponse = false - triggerEvent.DeploymentAppType = bean2.ArgoCd - triggerEvent.ManifestStorageType = bean2.ManifestStorageGit - case bean2.Helm: - triggerEvent.PerformChartPush = false - triggerEvent.PerformDeploymentOnCluster = true - triggerEvent.GetManifestInResponse = false - triggerEvent.DeploymentAppType = bean2.Helm - } - return triggerEvent -} - // write integration/unit test for each function -func (impl *WorkflowDagExecutorImpl) TriggerPipeline(overrideRequest *bean.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, triggerEvent bean.TriggerEvent, ctx context.Context) (releaseNo int, manifest []byte, err error) { - isRequestValid, err := impl.ValidateTriggerEvent(triggerEvent) - if !isRequestValid { - return releaseNo, manifest, err - } - - if triggerEvent.PerformChartPush { - //update workflow runner status, used in app workflow view - err = impl.UpdateCDWorkflowRunnerStatus(ctx, overrideRequest, triggerEvent.TriggerdAt, pipelineConfig.WorkflowInProgress, "") - if err != nil { - impl.logger.Errorw("error in updating the workflow runner status, createHelmAppForCdPipeline", "err", err) - return releaseNo, manifest, err - } - manifestPushTemplate, err := impl.BuildManifestPushTemplate(overrideRequest, valuesOverrideResponse, builtChartPath, &manifest) - if err != nil { - impl.logger.Errorw("error in building manifest push template", "err", err) - return releaseNo, manifest, err - } - manifestPushService := impl.GetManifestPushService(triggerEvent) - manifestPushResponse := manifestPushService.PushChart(manifestPushTemplate, ctx) - if manifestPushResponse.Error != nil { - impl.logger.Errorw("Error in pushing manifest to git", "err", err, "git_repo_url", manifestPushTemplate.RepoUrl) - return releaseNo, manifest, manifestPushResponse.Error - } - pipelineOverrideUpdateRequest := &chartConfig.PipelineOverride{ - Id: valuesOverrideResponse.PipelineOverride.Id, - GitHash: manifestPushResponse.CommitHash, - CommitTime: manifestPushResponse.CommitTime, - EnvConfigOverrideId: valuesOverrideResponse.EnvOverride.Id, - PipelineOverrideValues: valuesOverrideResponse.ReleaseOverrideJSON, - PipelineId: overrideRequest.PipelineId, - CiArtifactId: overrideRequest.CiArtifactId, - PipelineMergedValues: valuesOverrideResponse.MergedValues, - AuditLog: sql.AuditLog{UpdatedOn: triggerEvent.TriggerdAt, UpdatedBy: overrideRequest.UserId}, - } - _, span := otel.Tracer("orchestrator").Start(ctx, "pipelineOverrideRepository.Update") - err = impl.pipelineOverrideRepository.Update(pipelineOverrideUpdateRequest) - span.End() - } - - if triggerEvent.PerformDeploymentOnCluster { - err = impl.DeployApp(overrideRequest, valuesOverrideResponse, triggerEvent.TriggerdAt, ctx) - if err != nil { - impl.logger.Errorw("error in deploying app", "err", err) - return releaseNo, manifest, err - } - } - - go impl.WriteCDTriggerEvent(overrideRequest, valuesOverrideResponse.Artifact, valuesOverrideResponse.PipelineOverride.PipelineReleaseCounter, valuesOverrideResponse.PipelineOverride.Id) - - _, span := otel.Tracer("orchestrator").Start(ctx, "MarkImageScanDeployed") - _ = impl.MarkImageScanDeployed(overrideRequest.AppId, valuesOverrideResponse.EnvOverride.TargetEnvironment, valuesOverrideResponse.Artifact.ImageDigest, overrideRequest.ClusterId, valuesOverrideResponse.Artifact.ScanEnabled) - span.End() - - middleware.CdTriggerCounter.WithLabelValues(overrideRequest.AppName, overrideRequest.EnvName).Inc() - - return valuesOverrideResponse.PipelineOverride.PipelineReleaseCounter, manifest, nil - -} - -func (impl *WorkflowDagExecutorImpl) ValidateTriggerEvent(triggerEvent bean.TriggerEvent) (bool, error) { - - switch triggerEvent.DeploymentAppType { - case bean2.ArgoCd: - if !triggerEvent.PerformChartPush { - return false, errors2.New("For deployment type ArgoCd, PerformChartPush flag expected value = true, got false") - } - case bean2.Helm: - return true, nil - case bean2.GitOpsWithoutDeployment: - if triggerEvent.PerformDeploymentOnCluster { - return false, errors2.New("For deployment type GitOpsWithoutDeployment, PerformDeploymentOnCluster flag expected value = false, got value = true") - } - case bean2.ManifestDownload: - if triggerEvent.PerformChartPush { - return false, errors3.New("For deployment type ManifestDownload, PerformChartPush flag expected value = false, got true") - } - if triggerEvent.PerformDeploymentOnCluster { - return false, errors3.New("For deployment type ManifestDownload, PerformDeploymentOnCluster flag expected value = false, got true") - } - } - return true, nil - -} func (impl *WorkflowDagExecutorImpl) BuildManifestForTrigger(overrideRequest *bean.ValuesOverrideRequest, triggeredAt time.Time, ctx context.Context) (valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, err error) { @@ -3246,170 +2871,6 @@ func (impl *WorkflowDagExecutorImpl) CreateHistoriesForDeploymentTrigger(pipelin return nil } -func (impl *WorkflowDagExecutorImpl) BuildManifestPushTemplate(overrideRequest *bean.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, manifest *[]byte) (*bean4.ManifestPushTemplate, error) { - - manifestPushTemplate := &bean4.ManifestPushTemplate{ - WorkflowRunnerId: overrideRequest.WfrId, - AppId: overrideRequest.AppId, - ChartRefId: valuesOverrideResponse.EnvOverride.Chart.ChartRefId, - EnvironmentId: valuesOverrideResponse.EnvOverride.Environment.Id, - UserId: overrideRequest.UserId, - PipelineOverrideId: valuesOverrideResponse.PipelineOverride.Id, - AppName: overrideRequest.AppName, - TargetEnvironmentName: valuesOverrideResponse.EnvOverride.TargetEnvironment, - BuiltChartPath: builtChartPath, - BuiltChartBytes: manifest, - MergedValues: valuesOverrideResponse.MergedValues, - } - - manifestPushConfig, err := impl.manifestPushConfigRepository.GetManifestPushConfigByAppIdAndEnvId(overrideRequest.AppId, overrideRequest.EnvId) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in fetching manifest push config from db", "err", err) - return manifestPushTemplate, err - } - - if manifestPushConfig != nil { - if manifestPushConfig.StorageType == bean2.ManifestStorageGit { - // need to implement for git repo push - // currently manifest push config doesn't have git push config. Gitops config is derived from charts, chart_env_config_override and chart_ref table - } - } else { - manifestPushTemplate.ChartReferenceTemplate = valuesOverrideResponse.EnvOverride.Chart.ReferenceTemplate - manifestPushTemplate.ChartName = valuesOverrideResponse.EnvOverride.Chart.ChartName - manifestPushTemplate.ChartVersion = valuesOverrideResponse.EnvOverride.Chart.ChartVersion - manifestPushTemplate.ChartLocation = valuesOverrideResponse.EnvOverride.Chart.ChartLocation - manifestPushTemplate.RepoUrl = valuesOverrideResponse.EnvOverride.Chart.GitRepoUrl - } - return manifestPushTemplate, err -} - -func (impl *WorkflowDagExecutorImpl) GetManifestPushService(triggerEvent bean.TriggerEvent) app.ManifestPushService { - var manifestPushService app.ManifestPushService - if triggerEvent.ManifestStorageType == bean2.ManifestStorageGit { - manifestPushService = impl.gitOpsManifestPushService - } - return manifestPushService -} - -func (impl *WorkflowDagExecutorImpl) DeployApp(overrideRequest *bean.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, triggeredAt time.Time, ctx context.Context) error { - - if util.IsAcdApp(overrideRequest.DeploymentAppType) { - _, span := otel.Tracer("orchestrator").Start(ctx, "DeployArgocdApp") - err := impl.DeployArgocdApp(overrideRequest, valuesOverrideResponse, triggeredAt, ctx) - span.End() - if err != nil { - impl.logger.Errorw("error in deploying app on argocd", "err", err) - return err - } - } else if util.IsHelmApp(overrideRequest.DeploymentAppType) { - _, span := otel.Tracer("orchestrator").Start(ctx, "createHelmAppForCdPipeline") - _, err := impl.createHelmAppForCdPipeline(overrideRequest, valuesOverrideResponse, triggeredAt, ctx) - span.End() - if err != nil { - impl.logger.Errorw("error in creating or updating helm application for cd pipeline", "err", err) - return err - } - } - return nil -} - -func (impl *WorkflowDagExecutorImpl) WriteCDTriggerEvent(overrideRequest *bean.ValuesOverrideRequest, artifact *repository.CiArtifact, releaseId, pipelineOverrideId int) { - - event := impl.eventFactory.Build(util2.Trigger, &overrideRequest.PipelineId, overrideRequest.AppId, &overrideRequest.EnvId, util2.CD) - impl.logger.Debugw("event WriteCDTriggerEvent", "event", event) - event = impl.eventFactory.BuildExtraCDData(event, nil, pipelineOverrideId, bean.CD_WORKFLOW_TYPE_DEPLOY) - _, evtErr := impl.eventClient.WriteNotificationEvent(event) - if evtErr != nil { - impl.logger.Errorw("CD trigger event not sent", "error", evtErr) - } - deploymentEvent := app.DeploymentEvent{ - ApplicationId: overrideRequest.AppId, - EnvironmentId: overrideRequest.EnvId, //check for production Environment - ReleaseId: releaseId, - PipelineOverrideId: pipelineOverrideId, - TriggerTime: time.Now(), - CiArtifactId: overrideRequest.CiArtifactId, - } - ciPipelineMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(artifact.PipelineId) - if err != nil { - impl.logger.Errorw("error in ") - } - materialInfoMap, mErr := artifact.ParseMaterialInfo() - if mErr != nil { - impl.logger.Errorw("material info map error", mErr) - return - } - for _, ciPipelineMaterial := range ciPipelineMaterials { - hash := materialInfoMap[ciPipelineMaterial.GitMaterial.Url] - pipelineMaterialInfo := &app.PipelineMaterialInfo{PipelineMaterialId: ciPipelineMaterial.Id, CommitHash: hash} - deploymentEvent.PipelineMaterials = append(deploymentEvent.PipelineMaterials, pipelineMaterialInfo) - } - impl.logger.Infow("triggering deployment event", "event", deploymentEvent) - err = impl.eventClient.WriteNatsEvent(pubsub.CD_SUCCESS, deploymentEvent) - if err != nil { - impl.logger.Errorw("error in writing cd trigger event", "err", err) - } -} - -func (impl *WorkflowDagExecutorImpl) MarkImageScanDeployed(appId int, envId int, imageDigest string, clusterId int, isScanEnabled bool) error { - impl.logger.Debugw("mark image scan deployed for normal app, from cd auto or manual trigger", "imageDigest", imageDigest) - executionHistory, err := impl.imageScanHistoryRepository.FindByImageDigest(imageDigest) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in fetching execution history", "err", err) - return err - } - if executionHistory == nil || executionHistory.Id == 0 { - impl.logger.Errorw("no execution history found for digest", "digest", imageDigest) - return fmt.Errorf("no execution history found for digest - %s", imageDigest) - } - impl.logger.Debugw("mark image scan deployed for normal app, from cd auto or manual trigger", "executionHistory", executionHistory) - var ids []int - ids = append(ids, executionHistory.Id) - - ot, err := impl.imageScanDeployInfoRepository.FetchByAppIdAndEnvId(appId, envId, []string{security.ScanObjectType_APP}) - - if err == pg.ErrNoRows && !isScanEnabled { - //ignoring if no rows are found and scan is disabled - return nil - } - - if err != nil && err != pg.ErrNoRows { - return err - } else if err == pg.ErrNoRows && isScanEnabled { - imageScanDeployInfo := &security.ImageScanDeployInfo{ - ImageScanExecutionHistoryId: ids, - ScanObjectMetaId: appId, - ObjectType: security.ScanObjectType_APP, - EnvId: envId, - ClusterId: clusterId, - AuditLog: sql.AuditLog{ - CreatedOn: time.Now(), - CreatedBy: 1, - UpdatedOn: time.Now(), - UpdatedBy: 1, - }, - } - impl.logger.Debugw("mark image scan deployed for normal app, from cd auto or manual trigger", "imageScanDeployInfo", imageScanDeployInfo) - err = impl.imageScanDeployInfoRepository.Save(imageScanDeployInfo) - if err != nil { - impl.logger.Errorw("error in creating deploy info", "err", err) - } - } else { - // Updating Execution history for Latest Deployment to fetch out security Vulnerabilities for latest deployed info - if isScanEnabled { - ot.ImageScanExecutionHistoryId = ids - } else { - arr := []int{-1} - ot.ImageScanExecutionHistoryId = arr - } - err = impl.imageScanDeployInfoRepository.Update(ot) - if err != nil { - impl.logger.Errorw("error in updating deploy info for latest deployed image", "err", err) - } - } - return err -} - func (impl *WorkflowDagExecutorImpl) GetValuesOverrideForTrigger(overrideRequest *bean.ValuesOverrideRequest, triggeredAt time.Time, ctx context.Context) (*app.ValuesOverrideResponse, error) { if overrideRequest.DeploymentType == models.DEPLOYMENTTYPE_UNKNOWN { overrideRequest.DeploymentType = models.DEPLOYMENTTYPE_DEPLOY @@ -3562,224 +3023,6 @@ func getScopeForVariables(overrideRequest *bean.ValuesOverrideRequest, envOverri return scope } -func (impl *WorkflowDagExecutorImpl) DeployArgocdApp(overrideRequest *bean.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, triggeredAt time.Time, ctx context.Context) error { - - impl.logger.Debugw("new pipeline found", "pipeline", valuesOverrideResponse.Pipeline) - _, span := otel.Tracer("orchestrator").Start(ctx, "createArgoApplicationIfRequired") - name, err := impl.createArgoApplicationIfRequired(overrideRequest.AppId, valuesOverrideResponse.EnvOverride, valuesOverrideResponse.Pipeline, overrideRequest.UserId) - span.End() - if err != nil { - impl.logger.Errorw("acd application create error on cd trigger", "err", err, "req", overrideRequest) - return err - } - impl.logger.Debugw("argocd application created", "name", name) - - _, span = otel.Tracer("orchestrator").Start(ctx, "updateArgoPipeline") - updateAppInArgocd, err := impl.updateArgoPipeline(valuesOverrideResponse.Pipeline, valuesOverrideResponse.EnvOverride, ctx) - span.End() - if err != nil { - impl.logger.Errorw("error in updating argocd app ", "err", err) - return err - } - syncTime := time.Now() - err = impl.argoClientWrapperService.SyncArgoCDApplicationIfNeededAndRefresh(ctx, valuesOverrideResponse.Pipeline.DeploymentAppName) - if err != nil { - impl.logger.Errorw("error in getting argo application with normal refresh", "argoAppName", valuesOverrideResponse.Pipeline.DeploymentAppName) - return fmt.Errorf("%s. err: %s", ARGOCD_SYNC_ERROR, err.Error()) - } - if !impl.ACDConfig.ArgoCDAutoSyncEnabled { - timeline := &pipelineConfig.PipelineStatusTimeline{ - CdWorkflowRunnerId: overrideRequest.WfrId, - StatusTime: syncTime, - AuditLog: sql.AuditLog{ - CreatedBy: 1, - CreatedOn: time.Now(), - UpdatedBy: 1, - UpdatedOn: time.Now(), - }, - Status: pipelineConfig.TIMELINE_STATUS_ARGOCD_SYNC_COMPLETED, - StatusDetail: "argocd sync completed", - } - _, err, _ = impl.pipelineStatusTimelineService.SavePipelineStatusTimelineIfNotAlreadyPresent(overrideRequest.WfrId, timeline.Status, timeline, false) - if err != nil { - impl.logger.Errorw("error in saving pipeline status timeline", "err", err) - } - } - if updateAppInArgocd { - impl.logger.Debug("argo-cd successfully updated") - } else { - impl.logger.Debug("argo-cd failed to update, ignoring it") - } - return nil -} - -func (impl *WorkflowDagExecutorImpl) createArgoApplicationIfRequired(appId int, envConfigOverride *chartConfig.EnvConfigOverride, pipeline *pipelineConfig.Pipeline, userId int32) (string, error) { - //repo has been registered while helm create - chart, err := impl.chartRepository.FindLatestChartForAppByAppId(appId) - if err != nil { - impl.logger.Errorw("no chart found ", "app", appId) - return "", err - } - envModel, err := impl.envRepository.FindById(envConfigOverride.TargetEnvironment) - if err != nil { - return "", err - } - argoAppName := pipeline.DeploymentAppName - if pipeline.DeploymentAppCreated { - return argoAppName, nil - } else { - //create - appNamespace := envConfigOverride.Namespace - if appNamespace == "" { - appNamespace = "default" - } - namespace := argocdServer.DevtronInstalationNs - - appRequest := &argocdServer.AppTemplate{ - ApplicationName: argoAppName, - Namespace: namespace, - TargetNamespace: appNamespace, - TargetServer: envModel.Cluster.ServerUrl, - Project: "default", - ValuesFile: impl.getValuesFileForEnv(envModel.Id), - RepoPath: chart.ChartLocation, - RepoUrl: chart.GitRepoUrl, - AutoSyncEnabled: impl.ACDConfig.ArgoCDAutoSyncEnabled, - } - argoAppName, err := impl.argoK8sClient.CreateAcdApp(appRequest, envModel.Cluster, argocdServer.ARGOCD_APPLICATION_TEMPLATE) - if err != nil { - return "", err - } - //update cd pipeline to mark deployment app created - _, err = impl.updatePipeline(pipeline, userId) - if err != nil { - impl.logger.Errorw("error in update cd pipeline for deployment app created or not", "err", err) - return "", err - } - return argoAppName, nil - } -} - -func (impl *WorkflowDagExecutorImpl) createHelmAppForCdPipeline(overrideRequest *bean.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, triggeredAt time.Time, ctx context.Context) (bool, error) { - - pipeline := valuesOverrideResponse.Pipeline - envOverride := valuesOverrideResponse.EnvOverride - mergeAndSave := valuesOverrideResponse.MergedValues - - chartMetaData := &chart.Metadata{ - Name: pipeline.App.AppName, - Version: envOverride.Chart.ChartVersion, - } - referenceTemplatePath := path.Join(bean5.RefChartDirPath, envOverride.Chart.ReferenceTemplate) - - if util.IsHelmApp(pipeline.DeploymentAppType) { - referenceChartByte := envOverride.Chart.ReferenceChart - // here updating reference chart into database. - if len(envOverride.Chart.ReferenceChart) == 0 { - refChartByte, err := impl.chartTemplateService.GetByteArrayRefChart(chartMetaData, referenceTemplatePath) - if err != nil { - impl.logger.Errorw("ref chart commit error on cd trigger", "err", err, "req", overrideRequest) - return false, err - } - ch := envOverride.Chart - ch.ReferenceChart = refChartByte - ch.UpdatedOn = time.Now() - ch.UpdatedBy = overrideRequest.UserId - err = impl.chartRepository.Update(ch) - if err != nil { - impl.logger.Errorw("chart update error", "err", err, "req", overrideRequest) - return false, err - } - referenceChartByte = refChartByte - } - - releaseName := pipeline.DeploymentAppName - cluster := envOverride.Environment.Cluster - bearerToken := cluster.Config[util5.BearerToken] - clusterConfig := &gRPC.ClusterConfig{ - ClusterName: cluster.ClusterName, - Token: bearerToken, - ApiServerUrl: cluster.ServerUrl, - InsecureSkipTLSVerify: cluster.InsecureSkipTlsVerify, - } - if cluster.InsecureSkipTlsVerify == false { - clusterConfig.KeyData = cluster.Config[util5.TlsKey] - clusterConfig.CertData = cluster.Config[util5.CertData] - clusterConfig.CaData = cluster.Config[util5.CertificateAuthorityData] - } - releaseIdentifier := &gRPC.ReleaseIdentifier{ - ReleaseName: releaseName, - ReleaseNamespace: envOverride.Namespace, - ClusterConfig: clusterConfig, - } - - if pipeline.DeploymentAppCreated { - req := &gRPC.UpgradeReleaseRequest{ - ReleaseIdentifier: releaseIdentifier, - ValuesYaml: mergeAndSave, - HistoryMax: impl.helmAppService.GetRevisionHistoryMaxValue(bean6.SOURCE_DEVTRON_APP), - ChartContent: &gRPC.ChartContent{Content: referenceChartByte}, - } - if impl.appService.IsDevtronAsyncInstallModeEnabled(bean2.Helm) { - req.RunInCtx = true - } - // For cases where helm release was not found, kubelink will install the same configuration - updateApplicationResponse, err := impl.helmAppClient.UpdateApplication(ctx, req) - if err != nil { - impl.logger.Errorw("error in updating helm application for cd pipeline", "err", err) - if util.GetGRPCErrorDetailedMessage(err) == context.Canceled.Error() { - err = errors.New(pipelineConfig.NEW_DEPLOYMENT_INITIATED) - } - return false, err - } else { - impl.logger.Debugw("updated helm application", "response", updateApplicationResponse, "isSuccess", updateApplicationResponse.Success) - } - - } else { - - helmResponse, err := impl.helmInstallReleaseWithCustomChart(ctx, releaseIdentifier, referenceChartByte, mergeAndSave) - - // For connection related errors, no need to update the db - if err != nil && strings.Contains(err.Error(), "connection error") { - impl.logger.Errorw("error in helm install custom chart", "err", err) - return false, err - } - if util.GetGRPCErrorDetailedMessage(err) == context.Canceled.Error() { - err = errors.New(pipelineConfig.NEW_DEPLOYMENT_INITIATED) - } - - // IMP: update cd pipeline to mark deployment app created, even if helm install fails - // If the helm install fails, it still creates the app in failed state, so trying to - // re-create the app results in error from helm that cannot re-use name which is still in use - _, pgErr := impl.updatePipeline(pipeline, overrideRequest.UserId) - - if err != nil { - impl.logger.Errorw("error in helm install custom chart", "err", err) - - if pgErr != nil { - impl.logger.Errorw("failed to update deployment app created flag in pipeline table", "err", err) - } - return false, err - } - - if pgErr != nil { - impl.logger.Errorw("failed to update deployment app created flag in pipeline table", "err", err) - return false, err - } - - impl.logger.Debugw("received helm release response", "helmResponse", helmResponse, "isSuccess", helmResponse.Success) - } - - //update workflow runner status, used in app workflow view - err := impl.UpdateCDWorkflowRunnerStatus(ctx, overrideRequest, triggeredAt, pipelineConfig.WorkflowInProgress, "") - if err != nil { - impl.logger.Errorw("error in updating the workflow runner status, createHelmAppForCdPipeline", "err", err) - return false, err - } - } - return true, nil -} - func (impl *WorkflowDagExecutorImpl) GetDeploymentStrategyByTriggerType(overrideRequest *bean.ValuesOverrideRequest, ctx context.Context) (*chartConfig.PipelineStrategy, error) { strategy := &chartConfig.PipelineStrategy{} @@ -4375,59 +3618,6 @@ func (impl *WorkflowDagExecutorImpl) autoscalingCheckBeforeTrigger(ctx context.C return merged } -// update repoUrl, revision and argo app sync mode (auto/manual) if needed -func (impl *WorkflowDagExecutorImpl) updateArgoPipeline(pipeline *pipelineConfig.Pipeline, envOverride *chartConfig.EnvConfigOverride, ctx context.Context) (bool, error) { - if ctx == nil { - impl.logger.Errorw("err in syncing ACD, ctx is NULL", "pipelineName", pipeline.Name) - return false, nil - } - argoAppName := pipeline.DeploymentAppName - impl.logger.Infow("received payload, updateArgoPipeline", "appId", pipeline.AppId, "pipelineName", pipeline.Name, "envId", envOverride.TargetEnvironment, "argoAppName", argoAppName, "context", ctx) - argoApplication, err := impl.acdClient.Get(ctx, &application3.ApplicationQuery{Name: &argoAppName}) - if err != nil { - impl.logger.Errorw("no argo app exists", "app", argoAppName, "pipeline", pipeline.Name) - return false, err - } - //if status, ok:=status.FromError(err);ok{ - appStatus, _ := status2.FromError(err) - if appStatus.Code() == codes.OK { - impl.logger.Debugw("argo app exists", "app", argoAppName, "pipeline", pipeline.Name) - if argoApplication.Spec.Source.Path != envOverride.Chart.ChartLocation || argoApplication.Spec.Source.TargetRevision != "master" { - patchReq := v1alpha1.Application{Spec: v1alpha1.ApplicationSpec{Source: &v1alpha1.ApplicationSource{Path: envOverride.Chart.ChartLocation, RepoURL: envOverride.Chart.GitRepoUrl, TargetRevision: "master"}}} - reqbyte, err := json.Marshal(patchReq) - if err != nil { - impl.logger.Errorw("error in creating patch", "err", err) - } - reqString := string(reqbyte) - patchType := "merge" - _, err = impl.acdClient.Patch(ctx, &application3.ApplicationPatchRequest{Patch: &reqString, Name: &argoAppName, PatchType: &patchType}) - if err != nil { - impl.logger.Errorw("error in creating argo pipeline ", "name", pipeline.Name, "patch", string(reqbyte), "err", err) - return false, err - } - impl.logger.Debugw("pipeline update req ", "res", patchReq) - } else { - impl.logger.Debug("pipeline no need to update ") - } - err := impl.argoClientWrapperService.UpdateArgoCDSyncModeIfNeeded(ctx, argoApplication) - if err != nil { - impl.logger.Errorw("error in updating argocd sync mode", "err", err) - return false, err - } - return true, nil - } else if appStatus.Code() == codes.NotFound { - impl.logger.Errorw("argo app not found", "app", argoAppName, "pipeline", pipeline.Name) - return false, nil - } else { - impl.logger.Errorw("err in checking application on argoCD", "err", err, "pipeline", pipeline.Name) - return false, err - } -} - -func (impl *WorkflowDagExecutorImpl) getValuesFileForEnv(environmentId int) string { - return fmt.Sprintf("_%d-values.yaml", environmentId) //-{envId}-values.yaml -} - func (impl *WorkflowDagExecutorImpl) updatePipeline(pipeline *pipelineConfig.Pipeline, userId int32) (bool, error) { err := impl.pipelineRepository.SetDeploymentAppCreatedInPipeline(true, pipeline.Id, userId) if err != nil { @@ -4437,36 +3627,6 @@ func (impl *WorkflowDagExecutorImpl) updatePipeline(pipeline *pipelineConfig.Pip return true, nil } -// helmInstallReleaseWithCustomChart performs helm install with custom chart -func (impl *WorkflowDagExecutorImpl) helmInstallReleaseWithCustomChart(ctx context.Context, releaseIdentifier *gRPC.ReleaseIdentifier, referenceChartByte []byte, valuesYaml string) (*gRPC.HelmInstallCustomResponse, error) { - - helmInstallRequest := gRPC.HelmInstallCustomRequest{ - ValuesYaml: valuesYaml, - ChartContent: &gRPC.ChartContent{Content: referenceChartByte}, - ReleaseIdentifier: releaseIdentifier, - } - if impl.appService.IsDevtronAsyncInstallModeEnabled(bean2.Helm) { - helmInstallRequest.RunInCtx = true - } - // Request exec - return impl.helmAppClient.InstallReleaseWithCustomChart(ctx, &helmInstallRequest) -} - -type EnvironmentOverride struct { - Enabled bool `json:"enabled"` - EnvValues []*KeyValue `json:"envValues"` -} - -type KeyValue struct { - Key string `json:"key"` - Value string `json:"value"` -} - -func (conf *EnvironmentOverride) appendEnvironmentVariable(key, value string) { - item := &KeyValue{Key: key, Value: value} - conf.EnvValues = append(conf.EnvValues, item) -} - func (impl *WorkflowDagExecutorImpl) checkAndFixDuplicateReleaseNo(override *chartConfig.PipelineOverride) error { uniqueVerified := false @@ -4615,7 +3775,7 @@ func (impl *WorkflowDagExecutorImpl) UpdatePreviousQueuedRunnerStatus(cdWfrId, p return err } for _, cdWfr := range cdWfrs { - err = impl.MarkPipelineStatusTimelineFailed(cdWfr, errors.New(pipelineConfig.NEW_DEPLOYMENT_INITIATED)) + err = impl.pipelineStatusTimelineService.MarkPipelineStatusTimelineFailed(cdWfr.Id, pipelineConfig.NEW_DEPLOYMENT_INITIATED) if err != nil { impl.logger.Errorw("error updating CdPipelineStatusTimeline, UpdatePreviousQueuedRunnerStatus", "err", err) return err @@ -4630,91 +3790,7 @@ func (impl *WorkflowDagExecutorImpl) UpdatePreviousQueuedRunnerStatus(cdWfrId, p Pipeline: pipeline, } } - impl.UpdateTriggerCDMetricsOnFinish(cdWfr) - } - return nil -} - -func (impl *WorkflowDagExecutorImpl) UpdateCDWorkflowRunnerStatus(ctx context.Context, overrideRequest *bean.ValuesOverrideRequest, triggeredAt time.Time, status, message string) error { - // In case of terminal status update finished on time - isTerminalStatus := slices.Contains(pipelineConfig.WfrTerminalStatusList, status) - cdWfr, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(overrideRequest.WfrId) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("err on fetching cd workflow, UpdateCDWorkflowRunnerStatus", "err", err) - return err - } - cdWorkflowId := cdWfr.CdWorkflowId - - if cdWorkflowId == 0 { - cdWf := &pipelineConfig.CdWorkflow{ - CiArtifactId: overrideRequest.CiArtifactId, - PipelineId: overrideRequest.PipelineId, - AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: overrideRequest.UserId, UpdatedOn: triggeredAt, UpdatedBy: overrideRequest.UserId}, - } - err := impl.cdWorkflowRepository.SaveWorkFlow(ctx, cdWf) - if err != nil { - impl.logger.Errorw("err on updating cd workflow for status update, UpdateCDWorkflowRunnerStatus", "err", err) - return err - } - cdWorkflowId = cdWf.Id - runner := &pipelineConfig.CdWorkflowRunner{ - Id: cdWf.Id, - Name: overrideRequest.PipelineName, - WorkflowType: bean.CD_WORKFLOW_TYPE_DEPLOY, - ExecutorType: pipelineConfig.WORKFLOW_EXECUTOR_TYPE_AWF, - Status: status, - TriggeredBy: overrideRequest.UserId, - StartedOn: triggeredAt, - CdWorkflowId: cdWorkflowId, - AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: overrideRequest.UserId, UpdatedOn: triggeredAt, UpdatedBy: overrideRequest.UserId}, - } - if isTerminalStatus { - runner.FinishedOn = time.Now() - } - _, err = impl.cdWorkflowRepository.SaveWorkFlowRunner(runner) - if err != nil { - impl.logger.Errorw("err on updating cd workflow runner for status update, UpdateCDWorkflowRunnerStatus", "err", err) - return err - } - } else { - // if the current cdWfr status is already a terminal status and then don't update the status - // e.g: Status : Failed --> Progressing (not allowed) - if slices.Contains(pipelineConfig.WfrTerminalStatusList, cdWfr.Status) { - impl.logger.Warnw("deployment has already been terminated for workflow runner, UpdateCDWorkflowRunnerStatus", "workflowRunnerId", cdWfr.Id, "err", err) - return fmt.Errorf("deployment has already been terminated for workflow runner") - } - if status == pipelineConfig.WorkflowFailed { - err = impl.MarkPipelineStatusTimelineFailed(cdWfr, errors.New(message)) - if err != nil { - impl.logger.Errorw("error updating CdPipelineStatusTimeline", "err", err) - return err - } - } - cdWfr.Status = status - if isTerminalStatus { - cdWfr.FinishedOn = time.Now() - cdWfr.Message = message - } - cdWfr.UpdatedBy = overrideRequest.UserId - cdWfr.UpdatedOn = time.Now() - err = impl.cdWorkflowRepository.UpdateWorkFlowRunner(cdWfr) - if err != nil { - impl.logger.Errorw("error on update cd workflow runner, UpdateCDWorkflowRunnerStatus", "cdWfr", cdWfr, "err", err) - return err - } - } - if isTerminalStatus { - if cdWfr.CdWorkflow == nil { - pipeline, err := impl.pipelineRepository.FindById(overrideRequest.PipelineId) - if err != nil { - impl.logger.Errorw("error in fetching cd pipeline", "pipelineId", overrideRequest.PipelineId, "err", err) - return err - } - cdWfr.CdWorkflow = &pipelineConfig.CdWorkflow{ - Pipeline: pipeline, - } - } - impl.UpdateTriggerCDMetricsOnFinish(cdWfr) + util4.TriggerCDMetrics(pipelineConfig.GetTriggerMetricsFromRunnerObj(cdWfr), impl.config.ExposeCDMetrics) } return nil } diff --git a/pkg/workflow/cd/CdWorkflowCommonService.go b/pkg/workflow/cd/CdWorkflowCommonService.go new file mode 100644 index 0000000000..c53178f105 --- /dev/null +++ b/pkg/workflow/cd/CdWorkflowCommonService.go @@ -0,0 +1,132 @@ +package cd + +import ( + "context" + "fmt" + "github.com/devtron-labs/devtron/api/bean" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/pkg/app/status" + "github.com/devtron-labs/devtron/pkg/pipeline/types" + "github.com/devtron-labs/devtron/pkg/sql" + util4 "github.com/devtron-labs/devtron/util" + "github.com/go-pg/pg" + "go.uber.org/zap" + "slices" + "time" +) + +type CdWorkflowCommonService interface { + UpdateCDWorkflowRunnerStatus(ctx context.Context, overrideRequest *bean.ValuesOverrideRequest, triggeredAt time.Time, status, message string) error +} + +type CdWorkflowCommonServiceImpl struct { + logger *zap.SugaredLogger + cdWorkflowRepository pipelineConfig.CdWorkflowRepository + pipelineStatusTimelineService status.PipelineStatusTimelineService + + //TODO: remove below + config *types.CdConfig + pipelineRepository pipelineConfig.PipelineRepository +} + +func NewCdWorkflowCommonServiceImpl(logger *zap.SugaredLogger, + cdWorkflowRepository pipelineConfig.CdWorkflowRepository, + pipelineStatusTimelineService status.PipelineStatusTimelineService, + pipelineRepository pipelineConfig.PipelineRepository) (*CdWorkflowCommonServiceImpl, error) { + config, err := types.GetCdConfig() + if err != nil { + return nil, err + } + return &CdWorkflowCommonServiceImpl{ + logger: logger, + cdWorkflowRepository: cdWorkflowRepository, + pipelineStatusTimelineService: pipelineStatusTimelineService, + config: config, + pipelineRepository: pipelineRepository, + }, nil +} + +// TODO: remove bean.ValuesOverrideRequest +func (impl *CdWorkflowCommonServiceImpl) UpdateCDWorkflowRunnerStatus(ctx context.Context, overrideRequest *bean.ValuesOverrideRequest, triggeredAt time.Time, status, message string) error { + // In case of terminal status update finished on time + isTerminalStatus := slices.Contains(pipelineConfig.WfrTerminalStatusList, status) + cdWfr, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(overrideRequest.WfrId) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("err on fetching cd workflow, UpdateCDWorkflowRunnerStatus", "err", err) + return err + } + cdWorkflowId := cdWfr.CdWorkflowId + + if cdWorkflowId == 0 { + cdWf := &pipelineConfig.CdWorkflow{ + CiArtifactId: overrideRequest.CiArtifactId, + PipelineId: overrideRequest.PipelineId, + AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: overrideRequest.UserId, UpdatedOn: triggeredAt, UpdatedBy: overrideRequest.UserId}, + } + err := impl.cdWorkflowRepository.SaveWorkFlow(ctx, cdWf) + if err != nil { + impl.logger.Errorw("err on updating cd workflow for status update, UpdateCDWorkflowRunnerStatus", "err", err) + return err + } + cdWorkflowId = cdWf.Id + runner := &pipelineConfig.CdWorkflowRunner{ + Id: cdWf.Id, + Name: overrideRequest.PipelineName, + WorkflowType: bean.CD_WORKFLOW_TYPE_DEPLOY, + ExecutorType: pipelineConfig.WORKFLOW_EXECUTOR_TYPE_AWF, + Status: status, + TriggeredBy: overrideRequest.UserId, + StartedOn: triggeredAt, + CdWorkflowId: cdWorkflowId, + AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: overrideRequest.UserId, UpdatedOn: triggeredAt, UpdatedBy: overrideRequest.UserId}, + } + if isTerminalStatus { + runner.FinishedOn = time.Now() + } + _, err = impl.cdWorkflowRepository.SaveWorkFlowRunner(runner) + if err != nil { + impl.logger.Errorw("err on updating cd workflow runner for status update, UpdateCDWorkflowRunnerStatus", "err", err) + return err + } + } else { + // if the current cdWfr status is already a terminal status and then don't update the status + // e.g: Status : Failed --> Progressing (not allowed) + if slices.Contains(pipelineConfig.WfrTerminalStatusList, cdWfr.Status) { + impl.logger.Warnw("deployment has already been terminated for workflow runner, UpdateCDWorkflowRunnerStatus", "workflowRunnerId", cdWfr.Id, "err", err) + return fmt.Errorf("deployment has already been terminated for workflow runner") + } + if status == pipelineConfig.WorkflowFailed { + err = impl.pipelineStatusTimelineService.MarkPipelineStatusTimelineFailed(cdWfr.Id, message) + if err != nil { + impl.logger.Errorw("error updating CdPipelineStatusTimeline", "err", err) + return err + } + } + cdWfr.Status = status + if isTerminalStatus { + cdWfr.FinishedOn = time.Now() + cdWfr.Message = message + } + cdWfr.UpdatedBy = overrideRequest.UserId + cdWfr.UpdatedOn = time.Now() + err = impl.cdWorkflowRepository.UpdateWorkFlowRunner(cdWfr) + if err != nil { + impl.logger.Errorw("error on update cd workflow runner, UpdateCDWorkflowRunnerStatus", "cdWfr", cdWfr, "err", err) + return err + } + } + if isTerminalStatus { + if cdWfr.CdWorkflow == nil { + pipeline, err := impl.pipelineRepository.FindById(overrideRequest.PipelineId) + if err != nil { + impl.logger.Errorw("error in fetching cd pipeline", "pipelineId", overrideRequest.PipelineId, "err", err) + return err + } + cdWfr.CdWorkflow = &pipelineConfig.CdWorkflow{ + Pipeline: pipeline, + } + } + util4.TriggerCDMetrics(pipelineConfig.GetTriggerMetricsFromRunnerObj(cdWfr), impl.config.ExposeCDMetrics) + } + return nil +} diff --git a/pkg/workflow/cd/CdWorkflowRunnerService.go b/pkg/workflow/cd/CdWorkflowRunnerService.go new file mode 100644 index 0000000000..68a64cbd6a --- /dev/null +++ b/pkg/workflow/cd/CdWorkflowRunnerService.go @@ -0,0 +1,35 @@ +package cd + +import ( + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/pkg/workflow/cd/adapter" + "github.com/devtron-labs/devtron/pkg/workflow/cd/bean" + "go.uber.org/zap" +) + +type CdWorkflowRunnerService interface { + FindWorkflowRunnerById(wfrId int) (*bean.CdWorkflowRunnerDto, error) +} + +type CdWorkflowRunnerServiceImpl struct { + logger *zap.SugaredLogger + cdWorkflowRepository pipelineConfig.CdWorkflowRepository +} + +func NewCdWorkflowRunnerServiceImpl(logger *zap.SugaredLogger, + cdWorkflowRepository pipelineConfig.CdWorkflowRepository) *CdWorkflowRunnerServiceImpl { + return &CdWorkflowRunnerServiceImpl{ + logger: logger, + cdWorkflowRepository: cdWorkflowRepository, + } +} + +func (impl *CdWorkflowRunnerServiceImpl) FindWorkflowRunnerById(wfrId int) (*bean.CdWorkflowRunnerDto, error) { + cdWfr, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(wfrId) + if err != nil { + impl.logger.Errorw("error in getting cd workflow runner by id", "err", err, "id", wfrId) + return nil, err + } + return adapter.ConvertCdWorkflowRunnerDbObjToDto(cdWfr), nil + +} diff --git a/pkg/workflow/cd/CdWorkflowService.go b/pkg/workflow/cd/CdWorkflowService.go new file mode 100644 index 0000000000..592ab55f6b --- /dev/null +++ b/pkg/workflow/cd/CdWorkflowService.go @@ -0,0 +1,45 @@ +package cd + +import ( + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/pkg/workflow/cd/adapter" + "github.com/devtron-labs/devtron/pkg/workflow/cd/bean" + "go.uber.org/zap" +) + +type CdWorkflowService interface { + CheckIfLatestWf(pipelineId, cdWfId int) (latest bool, err error) + UpdateWorkFlow(dto *bean.CdWorkflowDto) error +} + +type CdWorkflowServiceImpl struct { + logger *zap.SugaredLogger + cdWorkflowRepository pipelineConfig.CdWorkflowRepository +} + +func NewCdWorkflowServiceImpl(logger *zap.SugaredLogger, + cdWorkflowRepository pipelineConfig.CdWorkflowRepository) *CdWorkflowServiceImpl { + return &CdWorkflowServiceImpl{ + logger: logger, + cdWorkflowRepository: cdWorkflowRepository, + } +} + +func (impl *CdWorkflowServiceImpl) CheckIfLatestWf(pipelineId, cdWfId int) (latest bool, err error) { + latest, err = impl.cdWorkflowRepository.IsLatestWf(pipelineId, cdWfId) + if err != nil { + impl.logger.Errorw("error in checking if wf is latest", "pipelineId", pipelineId, "cdWfId", cdWfId, "err", err) + return false, err + } + return latest, nil +} + +func (impl *CdWorkflowServiceImpl) UpdateWorkFlow(dto *bean.CdWorkflowDto) error { + dbObj := adapter.ConvertCdWorkflowDtoToDbObj(dto) + err := impl.cdWorkflowRepository.UpdateWorkFlow(dbObj) + if err != nil { + impl.logger.Errorw("error in updating workflow", "err", err, "req", dto) + return err + } + return nil +} diff --git a/pkg/workflow/cd/adapter/adapter.go b/pkg/workflow/cd/adapter/adapter.go new file mode 100644 index 0000000000..7ba4b5b76d --- /dev/null +++ b/pkg/workflow/cd/adapter/adapter.go @@ -0,0 +1,48 @@ +package adapter + +import ( + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/pkg/sql" + "github.com/devtron-labs/devtron/pkg/workflow/cd/bean" + "time" +) + +func ConvertCdWorkflowRunnerDbObjToDto(dbObj *pipelineConfig.CdWorkflowRunner) *bean.CdWorkflowRunnerDto { + newReferenceId := "" + if dbObj.ReferenceId != nil { + newReferenceId = *dbObj.ReferenceId + } + return &bean.CdWorkflowRunnerDto{ + Id: dbObj.Id, + Name: dbObj.Name, + WorkflowType: dbObj.WorkflowType, + ExecutorType: dbObj.ExecutorType, + Status: dbObj.Status, + PodStatus: dbObj.PodStatus, + Message: dbObj.Message, + StartedOn: dbObj.StartedOn, + FinishedOn: dbObj.FinishedOn, + Namespace: dbObj.Namespace, + LogLocation: dbObj.LogLocation, + TriggeredBy: dbObj.TriggeredBy, + CdWorkflowId: dbObj.CdWorkflowId, + PodName: dbObj.PodName, + BlobStorageEnabled: dbObj.BlobStorageEnabled, + RefCdWorkflowRunnerId: dbObj.RefCdWorkflowRunnerId, + ImagePathReservationIds: dbObj.ImagePathReservationIds, + ReferenceId: &newReferenceId, + } +} + +func ConvertCdWorkflowDtoToDbObj(dto *bean.CdWorkflowDto) *pipelineConfig.CdWorkflow { + return &pipelineConfig.CdWorkflow{ + Id: dto.Id, + CiArtifactId: dto.CiArtifactId, + PipelineId: dto.PipelineId, + WorkflowStatus: dto.WorkflowStatus, + AuditLog: sql.AuditLog{ //not handling for creation auditLog currently + UpdatedOn: time.Now(), + UpdatedBy: dto.UserId, + }, + } +} diff --git a/pkg/workflow/cd/bean/bean.go b/pkg/workflow/cd/bean/bean.go new file mode 100644 index 0000000000..f0413f843d --- /dev/null +++ b/pkg/workflow/cd/bean/bean.go @@ -0,0 +1,37 @@ +package bean + +import ( + "github.com/devtron-labs/devtron/api/bean" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "time" +) + +type CdWorkflowDto struct { + Id int `json:"id"` + CiArtifactId int `json:"ciArtifactId"` + PipelineId int `json:"pipelineId"` + WorkflowStatus pipelineConfig.WorkflowStatus `json:"workflowStatus"` + UserId int32 `json:"-"` +} + +type CdWorkflowRunnerDto struct { + Id int `json:"id"` + Name string `json:"name"` + WorkflowType bean.WorkflowType `json:"workflowType"` // pre,post,deploy + //TODO: extract from repo service layer + ExecutorType pipelineConfig.WorkflowExecutorType `json:"executorType"` // awf, system + Status string `json:"status"` + PodStatus string `json:"podStatus"` + Message string `json:"message"` + StartedOn time.Time `json:"startedOn"` + FinishedOn time.Time `json:"finishedOn"` + Namespace string `json:"namespace"` + LogLocation string `json:"logFilePath"` + TriggeredBy int32 `json:"triggeredBy"` + CdWorkflowId int `json:"cdWorkflowId"` + PodName string `json:"podName"` + BlobStorageEnabled bool `json:"blobStorageEnabled"` + RefCdWorkflowRunnerId int `json:"refCdWorkflowRunnerId"` + ImagePathReservationIds []int `json:"imagePathReservationIds"` + ReferenceId *string `json:"referenceId"` +} diff --git a/pkg/workflow/cd/wire_workflowCd.go b/pkg/workflow/cd/wire_workflowCd.go new file mode 100644 index 0000000000..38883bbb73 --- /dev/null +++ b/pkg/workflow/cd/wire_workflowCd.go @@ -0,0 +1,12 @@ +package cd + +import "github.com/google/wire" + +var CdWorkflowWireSet = wire.NewSet( + NewCdWorkflowCommonServiceImpl, + wire.Bind(new(CdWorkflowCommonService), new(*CdWorkflowCommonServiceImpl)), + NewCdWorkflowServiceImpl, + wire.Bind(new(CdWorkflowService), new(*CdWorkflowServiceImpl)), + NewCdWorkflowRunnerServiceImpl, + wire.Bind(new(CdWorkflowRunnerService), new(*CdWorkflowRunnerServiceImpl)), +) diff --git a/util/argo/ArgoUserService.go b/util/argo/ArgoUserService.go index 75715dd704..553007ed6b 100644 --- a/util/argo/ArgoUserService.go +++ b/util/argo/ArgoUserService.go @@ -40,6 +40,8 @@ type ArgoUserService interface { GetLatestDevtronArgoCdUserToken() (string, error) ValidateGitOpsAndGetOrUpdateArgoCdUserDetail() string GetOrUpdateArgoCdUserDetail() string + + BuildACDContext() (acdContext context.Context, err error) } type ArgoUserServiceImpl struct { @@ -73,6 +75,18 @@ func NewArgoUserServiceImpl(Logger *zap.SugaredLogger, clusterService cluster.Cl return argoUserServiceImpl, nil } +func (impl *ArgoUserServiceImpl) BuildACDContext() (acdContext context.Context, err error) { + //this part only accessible for acd apps hibernation, if acd configured it will fetch latest acdToken, else it will return error + acdToken, err := impl.GetLatestDevtronArgoCdUserToken() + if err != nil { + impl.logger.Errorw("error in getting acd token", "err", err) + return nil, err + } + ctx := context.Background() + ctx = context.WithValue(ctx, "token", acdToken) + return ctx, nil +} + func (impl *ArgoUserServiceImpl) ValidateGitOpsAndGetOrUpdateArgoCdUserDetail() string { isGitOpsConfigured, err := impl.gitOpsConfigReadService.IsGitOpsConfigured() if err != nil || !isGitOpsConfigured { diff --git a/util/argo/HelmUserService.go b/util/argo/HelmUserService.go index eb1221e61f..a852702a31 100644 --- a/util/argo/HelmUserService.go +++ b/util/argo/HelmUserService.go @@ -1,10 +1,12 @@ package argo import ( + "context" "errors" "go.uber.org/zap" ) +// TODO : remove this service completely type HelmUserServiceImpl struct { logger *zap.SugaredLogger } @@ -27,3 +29,7 @@ func (impl *HelmUserServiceImpl) ValidateGitOpsAndGetOrUpdateArgoCdUserDetail() func (impl *HelmUserServiceImpl) GetOrUpdateArgoCdUserDetail() string { return "" } + +func (impl *HelmUserServiceImpl) BuildACDContext() (acdContext context.Context, err error) { + return context.Background(), nil +} diff --git a/wire_gen.go b/wire_gen.go index e232c9c7ec..a0ed5b2727 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -111,10 +111,13 @@ import ( repository11 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/repository" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" + "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps" "github.com/devtron-labs/devtron/pkg/deploymentGroup" "github.com/devtron-labs/devtron/pkg/devtronResource" repository7 "github.com/devtron-labs/devtron/pkg/devtronResource/repository" "github.com/devtron-labs/devtron/pkg/dockerRegistry" + "github.com/devtron-labs/devtron/pkg/eventProcessor" + "github.com/devtron-labs/devtron/pkg/eventProcessor/in" "github.com/devtron-labs/devtron/pkg/externalLink" "github.com/devtron-labs/devtron/pkg/generateManifest" "github.com/devtron-labs/devtron/pkg/genericNotes" @@ -154,6 +157,7 @@ import ( "github.com/devtron-labs/devtron/pkg/variables/parsers" repository6 "github.com/devtron-labs/devtron/pkg/variables/repository" "github.com/devtron-labs/devtron/pkg/webhook/helm" + "github.com/devtron-labs/devtron/pkg/workflow/cd" util2 "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/argo" "github.com/devtron-labs/devtron/util/cron" @@ -457,7 +461,12 @@ func InitializeApp() (*App, error) { argoClientWrapperServiceImpl := argocdServer.NewArgoClientWrapperServiceImpl(sugaredLogger, applicationServiceClientImpl, acdConfig, repositoryServiceClientImpl) pipelineConfigListenerServiceImpl := pipeline.NewPipelineConfigListenerServiceImpl(sugaredLogger) imageDigestPolicyServiceImpl := imageDigestPolicy.NewImageDigestPolicyServiceImpl(sugaredLogger, qualifierMappingServiceImpl, devtronResourceSearchableKeyServiceImpl) - workflowDagExecutorImpl := pipeline.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, pubSubClientServiceImpl, appServiceImpl, workflowServiceImpl, ciArtifactRepositoryImpl, ciPipelineRepositoryImpl, materialRepositoryImpl, pipelineOverrideRepositoryImpl, userServiceImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, enforcerUtilImpl, eventSimpleFactoryImpl, eventRESTClientImpl, cvePolicyRepositoryImpl, imageScanResultRepositoryImpl, appWorkflowRepositoryImpl, prePostCdScriptHistoryServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineServiceImpl, ciTemplateRepositoryImpl, ciWorkflowRepositoryImpl, appLabelRepositoryImpl, clientImpl, pipelineStageServiceImpl, k8sCommonServiceImpl, globalPluginServiceImpl, pluginInputVariableParserImpl, scopedVariableCMCSManagerImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineStrategyHistoryServiceImpl, manifestPushConfigRepositoryImpl, gitOpsManifestPushServiceImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, appCrudOperationServiceImpl, pipelineConfigRepositoryImpl, dockerRegistryIpsConfigServiceImpl, chartRepositoryImpl, chartTemplateServiceImpl, pipelineStrategyHistoryRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, argoK8sClientImpl, configMapRepositoryImpl, configMapHistoryRepositoryImpl, helmAppServiceImpl, helmAppClientImpl, envConfigOverrideRepositoryImpl, mergeUtil, applicationServiceClientImpl, argoClientWrapperServiceImpl, pipelineConfigListenerServiceImpl, customTagServiceImpl, acdConfig, deployedAppMetricsServiceImpl, chartRefServiceImpl, gitOpsConfigReadServiceImpl, gitOperationServiceImpl, imageDigestPolicyServiceImpl) + cdWorkflowCommonServiceImpl, err := cd.NewCdWorkflowCommonServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl, pipelineStatusTimelineServiceImpl, pipelineRepositoryImpl) + if err != nil { + return nil, err + } + triggerServiceImpl := devtronApps.NewTriggerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, argoK8sClientImpl, acdConfig, applicationServiceClientImpl, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, appServiceImpl, helmAppServiceImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl) + workflowDagExecutorImpl := pipeline.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, pubSubClientServiceImpl, appServiceImpl, workflowServiceImpl, ciArtifactRepositoryImpl, ciPipelineRepositoryImpl, materialRepositoryImpl, pipelineOverrideRepositoryImpl, userServiceImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, enforcerUtilImpl, eventSimpleFactoryImpl, eventRESTClientImpl, cvePolicyRepositoryImpl, imageScanResultRepositoryImpl, appWorkflowRepositoryImpl, prePostCdScriptHistoryServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineServiceImpl, ciTemplateRepositoryImpl, ciWorkflowRepositoryImpl, appLabelRepositoryImpl, clientImpl, pipelineStageServiceImpl, k8sCommonServiceImpl, globalPluginServiceImpl, pluginInputVariableParserImpl, scopedVariableCMCSManagerImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineStrategyHistoryServiceImpl, manifestPushConfigRepositoryImpl, gitOpsManifestPushServiceImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, appCrudOperationServiceImpl, pipelineConfigRepositoryImpl, dockerRegistryIpsConfigServiceImpl, chartRepositoryImpl, chartTemplateServiceImpl, pipelineStrategyHistoryRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, argoK8sClientImpl, configMapRepositoryImpl, configMapHistoryRepositoryImpl, helmAppServiceImpl, helmAppClientImpl, envConfigOverrideRepositoryImpl, mergeUtil, applicationServiceClientImpl, argoClientWrapperServiceImpl, pipelineConfigListenerServiceImpl, customTagServiceImpl, acdConfig, deployedAppMetricsServiceImpl, chartRefServiceImpl, imageDigestPolicyServiceImpl, cdWorkflowCommonServiceImpl, triggerServiceImpl) deploymentGroupAppRepositoryImpl := repository.NewDeploymentGroupAppRepositoryImpl(sugaredLogger, db) deploymentGroupServiceImpl := deploymentGroup.NewDeploymentGroupServiceImpl(appRepositoryImpl, sugaredLogger, pipelineRepositoryImpl, ciPipelineRepositoryImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, deploymentGroupAppRepositoryImpl, ciArtifactRepositoryImpl, appWorkflowRepositoryImpl, workflowDagExecutorImpl) deploymentConfigServiceImpl := pipeline.NewDeploymentConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, pipelineRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, configMapHistoryServiceImpl, scopedVariableCMCSManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) @@ -822,6 +831,16 @@ func InitializeApp() (*App, error) { proxyRouterImpl := proxy.NewProxyRouterImpl(sugaredLogger, proxyConfig, enforcerImpl) muxRouter := router.NewMuxRouter(sugaredLogger, pipelineTriggerRouterImpl, pipelineConfigRouterImpl, appListingRouterImpl, environmentRouterImpl, clusterRouterImpl, webhookRouterImpl, userAuthRouterImpl, applicationRouterImpl, cdRouterImpl, gitProviderRouterImpl, gitHostRouterImpl, dockerRegRouterImpl, notificationRouterImpl, teamRouterImpl, gitWebhookHandlerImpl, workflowStatusUpdateHandlerImpl, applicationStatusHandlerImpl, ciEventHandlerImpl, pubSubClientServiceImpl, userRouterImpl, chartRefRouterImpl, configMapRouterImpl, appStoreRouterImpl, chartRepositoryRouterImpl, releaseMetricsRouterImpl, deploymentGroupRouterImpl, batchOperationRouterImpl, chartGroupRouterImpl, imageScanRouterImpl, policyRouterImpl, gitOpsConfigRouterImpl, dashboardRouterImpl, attributesRouterImpl, userAttributesRouterImpl, commonRouterImpl, grafanaRouterImpl, ssoLoginRouterImpl, telemetryRouterImpl, telemetryEventClientImplExtended, bulkUpdateRouterImpl, webhookListenerRouterImpl, appRouterImpl, coreAppRouterImpl, helmAppRouterImpl, k8sApplicationRouterImpl, pProfRouterImpl, deploymentConfigRouterImpl, dashboardTelemetryRouterImpl, commonDeploymentRouterImpl, externalLinkRouterImpl, globalPluginRouterImpl, moduleRouterImpl, serverRouterImpl, apiTokenRouterImpl, cdApplicationStatusUpdateHandlerImpl, k8sCapacityRouterImpl, webhookHelmRouterImpl, globalCMCSRouterImpl, userTerminalAccessRouterImpl, jobRouterImpl, ciStatusUpdateCronImpl, resourceGroupingRouterImpl, rbacRoleRouterImpl, scopedVariableRouterImpl, ciTriggerCronImpl, proxyRouterImpl) loggingMiddlewareImpl := util4.NewLoggingMiddlewareImpl(userServiceImpl) - mainApp := NewApp(muxRouter, sugaredLogger, sseSSE, syncedEnforcer, db, pubSubClientServiceImpl, sessionManager, posthogClient, loggingMiddlewareImpl) + cdWorkflowServiceImpl := cd.NewCdWorkflowServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) + cdWorkflowRunnerServiceImpl := cd.NewCdWorkflowRunnerServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) + workflowEventProcessorImpl, err := in.NewWorkflowEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowServiceImpl, cdWorkflowRunnerServiceImpl, workflowDagExecutorImpl, argoUserServiceImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl) + if err != nil { + return nil, err + } + centralEventProcessor, err := eventProcessor.NewCentralEventProcessor(workflowEventProcessorImpl, sugaredLogger) + if err != nil { + return nil, err + } + mainApp := NewApp(muxRouter, sugaredLogger, sseSSE, syncedEnforcer, db, pubSubClientServiceImpl, sessionManager, posthogClient, loggingMiddlewareImpl, centralEventProcessor) return mainApp, nil } From cd3703b9ef04eb44ffd22345858ccf6e29cb6247 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Wed, 31 Jan 2024 17:08:19 +0530 Subject: [PATCH 48/64] removed gitOpsRepoName fetch logic duplicacy --- api/router/pubsub/ApplicationStatusHandler.go | 14 ++++++-------- pkg/app/AppService.go | 13 +------------ wire_gen.go | 2 +- 3 files changed, 8 insertions(+), 21 deletions(-) diff --git a/api/router/pubsub/ApplicationStatusHandler.go b/api/router/pubsub/ApplicationStatusHandler.go index 3767a44f1e..e59165e701 100644 --- a/api/router/pubsub/ApplicationStatusHandler.go +++ b/api/router/pubsub/ApplicationStatusHandler.go @@ -24,6 +24,7 @@ import ( "fmt" "github.com/devtron-labs/common-lib/pubsub-lib/model" "github.com/devtron-labs/devtron/pkg/app" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" "k8s.io/utils/pointer" "time" @@ -56,12 +57,14 @@ type ApplicationStatusHandlerImpl struct { pipelineBuilder pipeline.PipelineBuilder pipelineRepository pipelineConfig.PipelineRepository installedAppRepository repository4.InstalledAppRepository + gitOpsConfigReadService config.GitOpsConfigReadService } func NewApplicationStatusHandlerImpl(logger *zap.SugaredLogger, pubsubClient *pubsub.PubSubClientServiceImpl, appService app.AppService, workflowDagExecutor pipeline.WorkflowDagExecutor, installedAppService service.InstalledAppService, appStoreDeploymentService service.AppStoreDeploymentService, pipelineBuilder pipeline.PipelineBuilder, - pipelineRepository pipelineConfig.PipelineRepository, installedAppRepository repository4.InstalledAppRepository) *ApplicationStatusHandlerImpl { + pipelineRepository pipelineConfig.PipelineRepository, installedAppRepository repository4.InstalledAppRepository, + gitOpsConfigReadService config.GitOpsConfigReadService) *ApplicationStatusHandlerImpl { appStatusUpdateHandlerImpl := &ApplicationStatusHandlerImpl{ logger: logger, pubsubClient: pubsubClient, @@ -72,6 +75,7 @@ func NewApplicationStatusHandlerImpl(logger *zap.SugaredLogger, pubsubClient *pu pipelineBuilder: pipelineBuilder, pipelineRepository: pipelineRepository, installedAppRepository: installedAppRepository, + gitOpsConfigReadService: gitOpsConfigReadService, } err := appStatusUpdateHandlerImpl.Subscribe() if err != nil { @@ -119,13 +123,7 @@ func (impl *ApplicationStatusHandlerImpl) Subscribe() error { impl.logger.Errorw("error in getting all gitops deployment app names from installed_apps ", "err", err) return } - var devtronGitOpsAppName string - gitOpsRepoPrefix := impl.appService.GetGitOpsRepoPrefix() - if len(gitOpsRepoPrefix) > 0 { - devtronGitOpsAppName = fmt.Sprintf("%s-%s", gitOpsRepoPrefix, app.ObjectMeta.Name) - } else { - devtronGitOpsAppName = app.ObjectMeta.Name - } + devtronGitOpsAppName := impl.gitOpsConfigReadService.GetGitOpsRepoName(app.ObjectMeta.Name) if slices.Contains(gitOpsDeployedAppNames, devtronGitOpsAppName) { // app found in installed_apps table hence setting flag to true isAppStoreApplication = true diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index 10e61a0a28..5286557974 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -137,7 +137,6 @@ type AppService interface { //MarkImageScanDeployed(appId int, envId int, imageDigest string, clusterId int, isScanEnabled bool) error UpdateDeploymentStatusForGitOpsPipelines(app *v1alpha1.Application, statusTime time.Time, isAppStore bool) (bool, bool, *chartConfig.PipelineOverride, error) WriteCDSuccessEvent(appId int, envId int, override *chartConfig.PipelineOverride) - GetGitOpsRepoPrefix() string //GetValuesOverrideForTrigger(overrideRequest *bean.ValuesOverrideRequest, triggeredAt time.Time, ctx context.Context) (*ValuesOverrideResponse, error) //GetEnvOverrideByTriggerType(overrideRequest *bean.ValuesOverrideRequest, triggeredAt time.Time, ctx context.Context) (*chartConfig.EnvConfigOverride, error) //GetAppMetricsByTriggerType(overrideRequest *bean.ValuesOverrideRequest, ctx context.Context) (bool, error) @@ -440,13 +439,7 @@ func (impl *AppServiceImpl) CheckIfPipelineUpdateEventIsValidForAppStore(gitOpsA for _, item := range gitOpsAppNameAndInstalledAppId { gitOpsAppNameAndInstalledAppMapping[item.GitOpsAppName] = &item.InstalledAppId } - var devtronAcdAppName string - if len(impl.globalEnvVariables.GitOpsRepoPrefix) > 0 { - devtronAcdAppName = fmt.Sprintf("%s-%s", impl.globalEnvVariables.GitOpsRepoPrefix, gitOpsAppName) - } else { - devtronAcdAppName = gitOpsAppName - } - + devtronAcdAppName := impl.gitOpsConfigReadService.GetGitOpsRepoName(gitOpsAppName) if gitOpsAppNameAndInstalledAppMapping[devtronAcdAppName] != nil { installedAppId = *gitOpsAppNameAndInstalledAppMapping[devtronAcdAppName] } @@ -1251,10 +1244,6 @@ const nameOverride = "nameOverride" const enabled = "enabled" const replicaCount = "replicaCount" -func (impl *AppServiceImpl) GetGitOpsRepoPrefix() string { - return impl.globalEnvVariables.GitOpsRepoPrefix -} - func (impl *AppServiceImpl) IsDevtronAsyncInstallModeEnabled(deploymentAppType string) bool { return impl.appStatusConfig.EnableAsyncInstallDevtronChart && deploymentAppType == bean4.Helm diff --git a/wire_gen.go b/wire_gen.go index a0ed5b2727..ce7929bb42 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -646,7 +646,7 @@ func InitializeApp() (*App, error) { teamRouterImpl := team2.NewTeamRouterImpl(teamRestHandlerImpl) gitWebhookHandlerImpl := pubsub.NewGitWebhookHandler(sugaredLogger, pubSubClientServiceImpl, gitWebhookServiceImpl) workflowStatusUpdateHandlerImpl := pubsub.NewWorkflowStatusUpdateHandlerImpl(sugaredLogger, pubSubClientServiceImpl, ciHandlerImpl, cdHandlerImpl, eventSimpleFactoryImpl, eventRESTClientImpl, cdWorkflowRepositoryImpl) - applicationStatusHandlerImpl := pubsub.NewApplicationStatusHandlerImpl(sugaredLogger, pubSubClientServiceImpl, appServiceImpl, workflowDagExecutorImpl, installedAppServiceImpl, appStoreDeploymentServiceImpl, pipelineBuilderImpl, pipelineRepositoryImpl, installedAppRepositoryImpl) + applicationStatusHandlerImpl := pubsub.NewApplicationStatusHandlerImpl(sugaredLogger, pubSubClientServiceImpl, appServiceImpl, workflowDagExecutorImpl, installedAppServiceImpl, appStoreDeploymentServiceImpl, pipelineBuilderImpl, pipelineRepositoryImpl, installedAppRepositoryImpl, gitOpsConfigReadServiceImpl) roleGroupServiceImpl := user.NewRoleGroupServiceImpl(userAuthRepositoryImpl, sugaredLogger, userRepositoryImpl, roleGroupRepositoryImpl, userCommonServiceImpl) userRestHandlerImpl := user2.NewUserRestHandlerImpl(userServiceImpl, validate, sugaredLogger, enforcerImpl, roleGroupServiceImpl, userCommonServiceImpl) userRouterImpl := user2.NewUserRouterImpl(userRestHandlerImpl) From f284a777c0312469959dffaa6fb4d223cee5e37e Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Wed, 31 Jan 2024 18:43:30 +0530 Subject: [PATCH 49/64] minor change for cd trigger method --- .../trigger/devtronApps/TriggerService.go | 56 ++++++++++++------- pkg/pipeline/WorkflowDagExecutor.go | 19 +------ 2 files changed, 39 insertions(+), 36 deletions(-) diff --git a/pkg/deployment/trigger/devtronApps/TriggerService.go b/pkg/deployment/trigger/devtronApps/TriggerService.go index 455985b8da..f33ac80415 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService.go +++ b/pkg/deployment/trigger/devtronApps/TriggerService.go @@ -47,8 +47,8 @@ import ( ) type TriggerService interface { - TriggerPipeline(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, - builtChartPath string, triggerEvent bean.TriggerEvent, ctx context.Context) (releaseNo int, manifest []byte, err error) + TriggerRelease(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, + builtChartPath string, ctx context.Context, triggeredAt time.Time, triggeredBy int32) (releaseNo int, manifest []byte, err error) } type TriggerServiceImpl struct { @@ -128,7 +128,23 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, cdWorkflowCommonService cd } } -func (impl *TriggerServiceImpl) TriggerPipeline(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, triggerEvent bean.TriggerEvent, ctx context.Context) (releaseNo int, manifest []byte, err error) { +// TriggerRelease will trigger Install/Upgrade request for Devtron App releases synchronously +func (impl *TriggerServiceImpl) TriggerRelease(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, + builtChartPath string, ctx context.Context, triggeredAt time.Time, triggeredBy int32) (releaseNo int, manifest []byte, err error) { + // Handling for auto trigger + if overrideRequest.UserId == 0 { + overrideRequest.UserId = triggeredBy + } + triggerEvent := helper.GetTriggerEvent(overrideRequest.DeploymentAppType, triggeredAt, triggeredBy) + releaseNo, manifest, err = impl.triggerPipeline(overrideRequest, valuesOverrideResponse, builtChartPath, triggerEvent, ctx) + if err != nil { + return 0, manifest, err + } + return releaseNo, manifest, nil +} + +func (impl *TriggerServiceImpl) triggerPipeline(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, + builtChartPath string, triggerEvent bean.TriggerEvent, ctx context.Context) (releaseNo int, manifest []byte, err error) { isRequestValid, err := helper.ValidateTriggerEvent(triggerEvent) if !isRequestValid { return releaseNo, manifest, err @@ -141,12 +157,12 @@ func (impl *TriggerServiceImpl) TriggerPipeline(overrideRequest *bean3.ValuesOve impl.logger.Errorw("error in updating the workflow runner status, createHelmAppForCdPipeline", "err", err) return releaseNo, manifest, err } - manifestPushTemplate, err := impl.BuildManifestPushTemplate(overrideRequest, valuesOverrideResponse, builtChartPath, &manifest) + manifestPushTemplate, err := impl.buildManifestPushTemplate(overrideRequest, valuesOverrideResponse, builtChartPath, &manifest) if err != nil { impl.logger.Errorw("error in building manifest push template", "err", err) return releaseNo, manifest, err } - manifestPushService := impl.GetManifestPushService(triggerEvent) + manifestPushService := impl.getManifestPushService(triggerEvent) manifestPushResponse := manifestPushService.PushChart(manifestPushTemplate, ctx) if manifestPushResponse.Error != nil { impl.logger.Errorw("Error in pushing manifest to git", "err", err, "git_repo_url", manifestPushTemplate.RepoUrl) @@ -169,17 +185,17 @@ func (impl *TriggerServiceImpl) TriggerPipeline(overrideRequest *bean3.ValuesOve } if triggerEvent.PerformDeploymentOnCluster { - err = impl.DeployApp(overrideRequest, valuesOverrideResponse, triggerEvent.TriggerdAt, ctx) + err = impl.deployApp(overrideRequest, valuesOverrideResponse, triggerEvent.TriggerdAt, ctx) if err != nil { impl.logger.Errorw("error in deploying app", "err", err) return releaseNo, manifest, err } } - go impl.WriteCDTriggerEvent(overrideRequest, valuesOverrideResponse.Artifact, valuesOverrideResponse.PipelineOverride.PipelineReleaseCounter, valuesOverrideResponse.PipelineOverride.Id) + go impl.writeCDTriggerEvent(overrideRequest, valuesOverrideResponse.Artifact, valuesOverrideResponse.PipelineOverride.PipelineReleaseCounter, valuesOverrideResponse.PipelineOverride.Id) - _, span := otel.Tracer("orchestrator").Start(ctx, "MarkImageScanDeployed") - _ = impl.MarkImageScanDeployed(overrideRequest.AppId, valuesOverrideResponse.EnvOverride.TargetEnvironment, valuesOverrideResponse.Artifact.ImageDigest, overrideRequest.ClusterId, valuesOverrideResponse.Artifact.ScanEnabled) + _, span := otel.Tracer("orchestrator").Start(ctx, "markImageScanDeployed") + _ = impl.markImageScanDeployed(overrideRequest.AppId, valuesOverrideResponse.EnvOverride.TargetEnvironment, valuesOverrideResponse.Artifact.ImageDigest, overrideRequest.ClusterId, valuesOverrideResponse.Artifact.ScanEnabled) span.End() middleware.CdTriggerCounter.WithLabelValues(overrideRequest.AppName, overrideRequest.EnvName).Inc() @@ -188,7 +204,7 @@ func (impl *TriggerServiceImpl) TriggerPipeline(overrideRequest *bean3.ValuesOve } -func (impl *TriggerServiceImpl) BuildManifestPushTemplate(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, manifest *[]byte) (*bean4.ManifestPushTemplate, error) { +func (impl *TriggerServiceImpl) buildManifestPushTemplate(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, manifest *[]byte) (*bean4.ManifestPushTemplate, error) { manifestPushTemplate := &bean4.ManifestPushTemplate{ WorkflowRunnerId: overrideRequest.WfrId, @@ -225,7 +241,7 @@ func (impl *TriggerServiceImpl) BuildManifestPushTemplate(overrideRequest *bean3 return manifestPushTemplate, err } -func (impl *TriggerServiceImpl) GetManifestPushService(triggerEvent bean.TriggerEvent) app.ManifestPushService { +func (impl *TriggerServiceImpl) getManifestPushService(triggerEvent bean.TriggerEvent) app.ManifestPushService { var manifestPushService app.ManifestPushService if triggerEvent.ManifestStorageType == bean2.ManifestStorageGit { manifestPushService = impl.gitOpsManifestPushService @@ -233,11 +249,12 @@ func (impl *TriggerServiceImpl) GetManifestPushService(triggerEvent bean.Trigger return manifestPushService } -func (impl *TriggerServiceImpl) DeployApp(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, triggeredAt time.Time, ctx context.Context) error { +func (impl *TriggerServiceImpl) deployApp(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, + triggeredAt time.Time, ctx context.Context) error { if util.IsAcdApp(overrideRequest.DeploymentAppType) { - _, span := otel.Tracer("orchestrator").Start(ctx, "DeployArgocdApp") - err := impl.DeployArgocdApp(overrideRequest, valuesOverrideResponse, triggeredAt, ctx) + _, span := otel.Tracer("orchestrator").Start(ctx, "deployArgocdApp") + err := impl.deployArgocdApp(overrideRequest, valuesOverrideResponse, triggeredAt, ctx) span.End() if err != nil { impl.logger.Errorw("error in deploying app on argocd", "err", err) @@ -255,7 +272,8 @@ func (impl *TriggerServiceImpl) DeployApp(overrideRequest *bean3.ValuesOverrideR return nil } -func (impl *TriggerServiceImpl) createHelmAppForCdPipeline(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, triggeredAt time.Time, ctx context.Context) (bool, error) { +func (impl *TriggerServiceImpl) createHelmAppForCdPipeline(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, + triggeredAt time.Time, ctx context.Context) (bool, error) { pipeline := valuesOverrideResponse.Pipeline envOverride := valuesOverrideResponse.EnvOverride @@ -375,7 +393,7 @@ func (impl *TriggerServiceImpl) createHelmAppForCdPipeline(overrideRequest *bean return true, nil } -func (impl *TriggerServiceImpl) DeployArgocdApp(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, triggeredAt time.Time, ctx context.Context) error { +func (impl *TriggerServiceImpl) deployArgocdApp(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, triggeredAt time.Time, ctx context.Context) error { impl.logger.Debugw("new pipeline found", "pipeline", valuesOverrideResponse.Pipeline) _, span := otel.Tracer("orchestrator").Start(ctx, "createArgoApplicationIfRequired") @@ -550,10 +568,10 @@ func (impl *TriggerServiceImpl) helmInstallReleaseWithCustomChart(ctx context.Co return impl.helmAppClient.InstallReleaseWithCustomChart(ctx, &helmInstallRequest) } -func (impl *TriggerServiceImpl) WriteCDTriggerEvent(overrideRequest *bean3.ValuesOverrideRequest, artifact *repository3.CiArtifact, releaseId, pipelineOverrideId int) { +func (impl *TriggerServiceImpl) writeCDTriggerEvent(overrideRequest *bean3.ValuesOverrideRequest, artifact *repository3.CiArtifact, releaseId, pipelineOverrideId int) { event := impl.eventFactory.Build(util2.Trigger, &overrideRequest.PipelineId, overrideRequest.AppId, &overrideRequest.EnvId, util2.CD) - impl.logger.Debugw("event WriteCDTriggerEvent", "event", event) + impl.logger.Debugw("event writeCDTriggerEvent", "event", event) event = impl.eventFactory.BuildExtraCDData(event, nil, pipelineOverrideId, bean3.CD_WORKFLOW_TYPE_DEPLOY) _, evtErr := impl.eventClient.WriteNotificationEvent(event) if evtErr != nil { @@ -588,7 +606,7 @@ func (impl *TriggerServiceImpl) WriteCDTriggerEvent(overrideRequest *bean3.Value } } -func (impl *TriggerServiceImpl) MarkImageScanDeployed(appId int, envId int, imageDigest string, clusterId int, isScanEnabled bool) error { +func (impl *TriggerServiceImpl) markImageScanDeployed(appId int, envId int, imageDigest string, clusterId int, isScanEnabled bool) error { impl.logger.Debugw("mark image scan deployed for normal app, from cd auto or manual trigger", "imageDigest", imageDigest) executionHistory, err := impl.imageScanHistoryRepository.FindByImageDigest(imageDigest) if err != nil && err != pg.ErrNoRows { diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index d053cf4539..00249ccdd0 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -27,7 +27,6 @@ import ( "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps" - "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/helper" bean7 "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" "github.com/devtron-labs/devtron/pkg/workflow/cd" "strconv" @@ -651,7 +650,7 @@ func (impl *WorkflowDagExecutorImpl) processDevtronAsyncHelmInstallRequest(CDAsy } _, span := otel.Tracer("orchestrator").Start(ctx, "appService.TriggerRelease") - releaseId, _, releaseErr := impl.TriggerRelease(overrideRequest, valuesOverrideResponse, builtChartPath, ctx, CDAsyncInstallNatsMessage.TriggeredAt, CDAsyncInstallNatsMessage.TriggeredBy) + releaseId, _, releaseErr := impl.cdTriggerService.TriggerRelease(overrideRequest, valuesOverrideResponse, builtChartPath, ctx, CDAsyncInstallNatsMessage.TriggeredAt, CDAsyncInstallNatsMessage.TriggeredBy) span.End() if releaseErr != nil { impl.handleAsyncTriggerReleaseError(releaseErr, cdWfr, overrideRequest, appIdentifier) @@ -2675,7 +2674,7 @@ func (impl *WorkflowDagExecutorImpl) HandleCDTriggerRelease(overrideRequest *bea impl.logger.Errorw("error in building merged manifest for trigger", "err", err) return releaseNo, manifest, err } - return impl.TriggerRelease(overrideRequest, valuesOverrideResponse, builtChartPath, ctx, triggeredAt, deployedBy) + return impl.cdTriggerService.TriggerRelease(overrideRequest, valuesOverrideResponse, builtChartPath, ctx, triggeredAt, deployedBy) } // TriggerHelmAsyncRelease will publish async helm Install/Upgrade request event for Devtron App releases @@ -2731,20 +2730,6 @@ func (impl *WorkflowDagExecutorImpl) TriggerHelmAsyncRelease(overrideRequest *be return 0, manifest, nil } -// TriggerRelease will trigger Install/Upgrade request for Devtron App releases synchronously -func (impl *WorkflowDagExecutorImpl) TriggerRelease(overrideRequest *bean.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, ctx context.Context, triggeredAt time.Time, triggeredBy int32) (releaseNo int, manifest []byte, err error) { - // Handling for auto trigger - if overrideRequest.UserId == 0 { - overrideRequest.UserId = triggeredBy - } - triggerEvent := helper.GetTriggerEvent(overrideRequest.DeploymentAppType, triggeredAt, triggeredBy) - releaseNo, manifest, err = impl.cdTriggerService.TriggerPipeline(overrideRequest, valuesOverrideResponse, builtChartPath, triggerEvent, ctx) - if err != nil { - return 0, manifest, err - } - return releaseNo, manifest, nil -} - func (impl *WorkflowDagExecutorImpl) TriggerCD(artifact *repository.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, triggeredAt time.Time) error { impl.logger.Debugw("automatic pipeline trigger attempt async", "artifactId", artifact.Id) From 2a9d94f077fd83824ec763812351b8e154328ea7 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Wed, 31 Jan 2024 19:07:49 +0530 Subject: [PATCH 50/64] removed redundant imports --- pkg/pipeline/WorkflowDagExecutor.go | 32 ----------------------------- wire_gen.go | 16 +++++++-------- 2 files changed, 8 insertions(+), 40 deletions(-) diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index 00249ccdd0..3d54028992 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -22,7 +22,6 @@ import ( "encoding/json" "fmt" bean6 "github.com/devtron-labs/devtron/api/helm-app/bean" - "github.com/devtron-labs/devtron/api/helm-app/gRPC" client2 "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" @@ -40,7 +39,6 @@ import ( "github.com/devtron-labs/common-lib/pubsub-lib/model" util5 "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/common-lib/utils/k8s/health" - "github.com/devtron-labs/devtron/client/argocdServer" "github.com/devtron-labs/devtron/client/argocdServer/application" application2 "github.com/devtron-labs/devtron/client/argocdServer/application" gitSensorClient "github.com/devtron-labs/devtron/client/gitSensor" @@ -163,30 +161,20 @@ type WorkflowDagExecutorImpl struct { deploymentTemplateHistoryService history2.DeploymentTemplateHistoryService configMapHistoryService history2.ConfigMapHistoryService pipelineStrategyHistoryService history2.PipelineStrategyHistoryService - manifestPushConfigRepository repository4.ManifestPushConfigRepository - gitOpsManifestPushService app.GitOpsPushService - ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository - imageScanHistoryRepository security.ImageScanHistoryRepository - imageScanDeployInfoRepository security.ImageScanDeployInfoRepository appCrudOperationService app.AppCrudOperationService pipelineConfigRepository chartConfig.PipelineConfigRepository dockerRegistryIpsConfigService dockerRegistry.DockerRegistryIpsConfigService chartRepository chartRepoRepository.ChartRepository - chartTemplateService util.ChartTemplateService strategyHistoryRepository repository3.PipelineStrategyHistoryRepository deploymentTemplateHistoryRepository repository3.DeploymentTemplateHistoryRepository - argoK8sClient argocdServer.ArgoK8sClient configMapRepository chartConfig.ConfigMapRepository configMapHistoryRepository repository3.ConfigMapHistoryRepository helmAppService client2.HelmAppService //TODO fix me next - helmAppClient gRPC.HelmAppClient //TODO refactoring: use helm app service instead environmentConfigRepository chartConfig.EnvConfigOverrideRepository mergeUtil *util.MergeUtil acdClient application2.ServiceClient - argoClientWrapperService argocdServer.ArgoClientWrapperService customTagService CustomTagService - ACDConfig *argocdServer.ACDConfig deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService chartRefService chartRef.ChartRefService imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService @@ -272,30 +260,20 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi deploymentTemplateHistoryService history2.DeploymentTemplateHistoryService, configMapHistoryService history2.ConfigMapHistoryService, pipelineStrategyHistoryService history2.PipelineStrategyHistoryService, - manifestPushConfigRepository repository4.ManifestPushConfigRepository, - gitOpsManifestPushService app.GitOpsPushService, - ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, - imageScanHistoryRepository security.ImageScanHistoryRepository, - imageScanDeployInfoRepository security.ImageScanDeployInfoRepository, appCrudOperationService app.AppCrudOperationService, pipelineConfigRepository chartConfig.PipelineConfigRepository, dockerRegistryIpsConfigService dockerRegistry.DockerRegistryIpsConfigService, chartRepository chartRepoRepository.ChartRepository, - chartTemplateService util.ChartTemplateService, strategyHistoryRepository repository3.PipelineStrategyHistoryRepository, deploymentTemplateHistoryRepository repository3.DeploymentTemplateHistoryRepository, - ArgoK8sClient argocdServer.ArgoK8sClient, configMapRepository chartConfig.ConfigMapRepository, configMapHistoryRepository repository3.ConfigMapHistoryRepository, helmAppService client2.HelmAppService, - helmAppClient gRPC.HelmAppClient, environmentConfigRepository chartConfig.EnvConfigOverrideRepository, mergeUtil *util.MergeUtil, acdClient application2.ServiceClient, - argoClientWrapperService argocdServer.ArgoClientWrapperService, pipelineConfigListenerService PipelineConfigListenerService, customTagService CustomTagService, - ACDConfig *argocdServer.ACDConfig, deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService, chartRefService chartRef.ChartRefService, imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService, @@ -342,29 +320,19 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi deploymentTemplateHistoryService: deploymentTemplateHistoryService, configMapHistoryService: configMapHistoryService, pipelineStrategyHistoryService: pipelineStrategyHistoryService, - manifestPushConfigRepository: manifestPushConfigRepository, - gitOpsManifestPushService: gitOpsManifestPushService, - ciPipelineMaterialRepository: ciPipelineMaterialRepository, - imageScanHistoryRepository: imageScanHistoryRepository, - imageScanDeployInfoRepository: imageScanDeployInfoRepository, appCrudOperationService: appCrudOperationService, pipelineConfigRepository: pipelineConfigRepository, dockerRegistryIpsConfigService: dockerRegistryIpsConfigService, chartRepository: chartRepository, - chartTemplateService: chartTemplateService, strategyHistoryRepository: strategyHistoryRepository, deploymentTemplateHistoryRepository: deploymentTemplateHistoryRepository, - argoK8sClient: ArgoK8sClient, configMapRepository: configMapRepository, configMapHistoryRepository: configMapHistoryRepository, helmAppService: helmAppService, - helmAppClient: helmAppClient, environmentConfigRepository: environmentConfigRepository, mergeUtil: mergeUtil, acdClient: acdClient, - argoClientWrapperService: argoClientWrapperService, customTagService: customTagService, - ACDConfig: ACDConfig, deployedAppMetricsService: deployedAppMetricsService, chartRefService: chartRefService, imageDigestPolicyService: imageDigestPolicyService, diff --git a/wire_gen.go b/wire_gen.go index ce7929bb42..4bb3bcdc70 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -444,10 +444,6 @@ func InitializeApp() (*App, error) { deploymentTemplateHistoryServiceImpl := history.NewDeploymentTemplateHistoryServiceImpl(sugaredLogger, deploymentTemplateHistoryRepositoryImpl, pipelineRepositoryImpl, chartRepositoryImpl, userServiceImpl, cdWorkflowRepositoryImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) pipelineStrategyHistoryRepositoryImpl := repository8.NewPipelineStrategyHistoryRepositoryImpl(sugaredLogger, db) pipelineStrategyHistoryServiceImpl := history.NewPipelineStrategyHistoryServiceImpl(sugaredLogger, pipelineStrategyHistoryRepositoryImpl, userServiceImpl) - manifestPushConfigRepositoryImpl := repository9.NewManifestPushConfigRepository(sugaredLogger, db) - gitOpsManifestPushServiceImpl := app2.NewGitOpsManifestPushServiceImpl(sugaredLogger, pipelineStatusTimelineServiceImpl, pipelineStatusTimelineRepositoryImpl, acdConfig, chartRefServiceImpl, gitOpsConfigReadServiceImpl, gitOperationServiceImpl) - imageScanHistoryRepositoryImpl := security.NewImageScanHistoryRepositoryImpl(db, sugaredLogger) - imageScanDeployInfoRepositoryImpl := security.NewImageScanDeployInfoRepositoryImpl(db, sugaredLogger) genericNoteRepositoryImpl := repository12.NewGenericNoteRepositoryImpl(db) genericNoteHistoryRepositoryImpl := repository12.NewGenericNoteHistoryRepositoryImpl(db) genericNoteHistoryServiceImpl := genericNotes.NewGenericNoteHistoryServiceImpl(genericNoteHistoryRepositoryImpl, sugaredLogger) @@ -456,17 +452,21 @@ func InitializeApp() (*App, error) { pipelineConfigRepositoryImpl := chartConfig.NewPipelineConfigRepository(db) ciTemplateOverrideRepositoryImpl := pipelineConfig.NewCiTemplateOverrideRepositoryImpl(db, sugaredLogger) dockerRegistryIpsConfigServiceImpl := dockerRegistry.NewDockerRegistryIpsConfigServiceImpl(sugaredLogger, dockerRegistryIpsConfigRepositoryImpl, k8sServiceImpl, clusterServiceImplExtended, ciPipelineRepositoryImpl, dockerArtifactStoreRepositoryImpl, ciTemplateOverrideRepositoryImpl) - argoK8sClientImpl := argocdServer.NewArgoK8sClientImpl(sugaredLogger) - repositoryServiceClientImpl := repository13.NewServiceClientImpl(sugaredLogger, argoCDConnectionManagerImpl) - argoClientWrapperServiceImpl := argocdServer.NewArgoClientWrapperServiceImpl(sugaredLogger, applicationServiceClientImpl, acdConfig, repositoryServiceClientImpl) pipelineConfigListenerServiceImpl := pipeline.NewPipelineConfigListenerServiceImpl(sugaredLogger) imageDigestPolicyServiceImpl := imageDigestPolicy.NewImageDigestPolicyServiceImpl(sugaredLogger, qualifierMappingServiceImpl, devtronResourceSearchableKeyServiceImpl) cdWorkflowCommonServiceImpl, err := cd.NewCdWorkflowCommonServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl, pipelineStatusTimelineServiceImpl, pipelineRepositoryImpl) if err != nil { return nil, err } + gitOpsManifestPushServiceImpl := app2.NewGitOpsManifestPushServiceImpl(sugaredLogger, pipelineStatusTimelineServiceImpl, pipelineStatusTimelineRepositoryImpl, acdConfig, chartRefServiceImpl, gitOpsConfigReadServiceImpl, gitOperationServiceImpl) + argoK8sClientImpl := argocdServer.NewArgoK8sClientImpl(sugaredLogger) + repositoryServiceClientImpl := repository13.NewServiceClientImpl(sugaredLogger, argoCDConnectionManagerImpl) + argoClientWrapperServiceImpl := argocdServer.NewArgoClientWrapperServiceImpl(sugaredLogger, applicationServiceClientImpl, acdConfig, repositoryServiceClientImpl) + imageScanHistoryRepositoryImpl := security.NewImageScanHistoryRepositoryImpl(db, sugaredLogger) + imageScanDeployInfoRepositoryImpl := security.NewImageScanDeployInfoRepositoryImpl(db, sugaredLogger) + manifestPushConfigRepositoryImpl := repository9.NewManifestPushConfigRepository(sugaredLogger, db) triggerServiceImpl := devtronApps.NewTriggerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, argoK8sClientImpl, acdConfig, applicationServiceClientImpl, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, appServiceImpl, helmAppServiceImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl) - workflowDagExecutorImpl := pipeline.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, pubSubClientServiceImpl, appServiceImpl, workflowServiceImpl, ciArtifactRepositoryImpl, ciPipelineRepositoryImpl, materialRepositoryImpl, pipelineOverrideRepositoryImpl, userServiceImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, enforcerUtilImpl, eventSimpleFactoryImpl, eventRESTClientImpl, cvePolicyRepositoryImpl, imageScanResultRepositoryImpl, appWorkflowRepositoryImpl, prePostCdScriptHistoryServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineServiceImpl, ciTemplateRepositoryImpl, ciWorkflowRepositoryImpl, appLabelRepositoryImpl, clientImpl, pipelineStageServiceImpl, k8sCommonServiceImpl, globalPluginServiceImpl, pluginInputVariableParserImpl, scopedVariableCMCSManagerImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineStrategyHistoryServiceImpl, manifestPushConfigRepositoryImpl, gitOpsManifestPushServiceImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, appCrudOperationServiceImpl, pipelineConfigRepositoryImpl, dockerRegistryIpsConfigServiceImpl, chartRepositoryImpl, chartTemplateServiceImpl, pipelineStrategyHistoryRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, argoK8sClientImpl, configMapRepositoryImpl, configMapHistoryRepositoryImpl, helmAppServiceImpl, helmAppClientImpl, envConfigOverrideRepositoryImpl, mergeUtil, applicationServiceClientImpl, argoClientWrapperServiceImpl, pipelineConfigListenerServiceImpl, customTagServiceImpl, acdConfig, deployedAppMetricsServiceImpl, chartRefServiceImpl, imageDigestPolicyServiceImpl, cdWorkflowCommonServiceImpl, triggerServiceImpl) + workflowDagExecutorImpl := pipeline.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, pubSubClientServiceImpl, appServiceImpl, workflowServiceImpl, ciArtifactRepositoryImpl, ciPipelineRepositoryImpl, materialRepositoryImpl, pipelineOverrideRepositoryImpl, userServiceImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, enforcerUtilImpl, eventSimpleFactoryImpl, eventRESTClientImpl, cvePolicyRepositoryImpl, imageScanResultRepositoryImpl, appWorkflowRepositoryImpl, prePostCdScriptHistoryServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineServiceImpl, ciTemplateRepositoryImpl, ciWorkflowRepositoryImpl, appLabelRepositoryImpl, clientImpl, pipelineStageServiceImpl, k8sCommonServiceImpl, globalPluginServiceImpl, pluginInputVariableParserImpl, scopedVariableCMCSManagerImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineStrategyHistoryServiceImpl, appCrudOperationServiceImpl, pipelineConfigRepositoryImpl, dockerRegistryIpsConfigServiceImpl, chartRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, configMapRepositoryImpl, configMapHistoryRepositoryImpl, helmAppServiceImpl, envConfigOverrideRepositoryImpl, mergeUtil, applicationServiceClientImpl, pipelineConfigListenerServiceImpl, customTagServiceImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, imageDigestPolicyServiceImpl, cdWorkflowCommonServiceImpl, triggerServiceImpl) deploymentGroupAppRepositoryImpl := repository.NewDeploymentGroupAppRepositoryImpl(sugaredLogger, db) deploymentGroupServiceImpl := deploymentGroup.NewDeploymentGroupServiceImpl(appRepositoryImpl, sugaredLogger, pipelineRepositoryImpl, ciPipelineRepositoryImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, deploymentGroupAppRepositoryImpl, ciArtifactRepositoryImpl, appWorkflowRepositoryImpl, workflowDagExecutorImpl) deploymentConfigServiceImpl := pipeline.NewDeploymentConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, pipelineRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, configMapHistoryServiceImpl, scopedVariableCMCSManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) From f41673abb78e6cca2379b082fcdf2f5e47c2010f Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Wed, 31 Jan 2024 19:46:50 +0530 Subject: [PATCH 51/64] extracted deployment bulk trigger publish event logic from workflowDag --- pkg/deploymentGroup/DeploymentGroupService.go | 14 ++-- .../out/WorkflowEventPublishService.go | 72 +++++++++++++++++++ pkg/eventProcessor/out/bean/bean.go | 6 ++ .../out/wire_eventProcessorOut.go | 8 +++ pkg/eventProcessor/wire_eventProcessor.go | 2 + .../AppDeploymentTypeChangeManager.go | 50 ++++++------- pkg/pipeline/WorkflowDagExecutor.go | 50 ------------- wire_gen.go | 9 ++- 8 files changed, 129 insertions(+), 82 deletions(-) create mode 100644 pkg/eventProcessor/out/WorkflowEventPublishService.go create mode 100644 pkg/eventProcessor/out/bean/bean.go create mode 100644 pkg/eventProcessor/out/wire_eventProcessorOut.go diff --git a/pkg/deploymentGroup/DeploymentGroupService.go b/pkg/deploymentGroup/DeploymentGroupService.go index 3ceb1357a2..f5bf07574f 100644 --- a/pkg/deploymentGroup/DeploymentGroupService.go +++ b/pkg/deploymentGroup/DeploymentGroupService.go @@ -22,6 +22,8 @@ import ( "fmt" "github.com/devtron-labs/devtron/internal/sql/repository/app" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/eventProcessor/out" + bean2 "github.com/devtron-labs/devtron/pkg/eventProcessor/out/bean" "strings" "time" @@ -110,7 +112,7 @@ type DeploymentGroupServiceImpl struct { deploymentGroupAppRepository repository.DeploymentGroupAppRepository ciArtifactRepository repository.CiArtifactRepository appWorkflowRepository appWorkflow.AppWorkflowRepository - workflowDagExecutor pipeline.WorkflowDagExecutor + workflowEventPublishService out.WorkflowEventPublishService } func NewDeploymentGroupServiceImpl(appRepository app.AppRepository, logger *zap.SugaredLogger, @@ -119,7 +121,7 @@ func NewDeploymentGroupServiceImpl(appRepository app.AppRepository, logger *zap. deploymentGroupAppRepository repository.DeploymentGroupAppRepository, ciArtifactRepository repository.CiArtifactRepository, appWorkflowRepository appWorkflow.AppWorkflowRepository, - workflowDagExecutor pipeline.WorkflowDagExecutor) *DeploymentGroupServiceImpl { + workflowEventPublishService out.WorkflowEventPublishService) *DeploymentGroupServiceImpl { return &DeploymentGroupServiceImpl{ appRepository: appRepository, logger: logger, @@ -130,7 +132,7 @@ func NewDeploymentGroupServiceImpl(appRepository app.AppRepository, logger *zap. deploymentGroupAppRepository: deploymentGroupAppRepository, ciArtifactRepository: ciArtifactRepository, appWorkflowRepository: appWorkflowRepository, - workflowDagExecutor: workflowDagExecutor, + workflowEventPublishService: workflowEventPublishService, } } @@ -481,7 +483,7 @@ func (impl *DeploymentGroupServiceImpl) TriggerReleaseForDeploymentGroup(trigger impl.logger.Errorw("no cdPipelines found", "req", triggerRequest) return nil, fmt.Errorf("no cdPipelines found corresponding to deployment group %d", triggerRequest.DeploymentGroupId) } - var requests []*pipeline.BulkTriggerRequest + var requests []*bean2.BulkTriggerRequest ciArtefactMapping := make(map[int]*repository.CiArtifact) for _, ciArtefact := range ciArtifacts { ciArtefactMapping[ciArtefact.PipelineId] = ciArtefact @@ -489,7 +491,7 @@ func (impl *DeploymentGroupServiceImpl) TriggerReleaseForDeploymentGroup(trigger for _, cdPipeline := range cdPipelines { if val, ok := ciArtefactMapping[cdPipeline.CiPipelineId]; ok { //do something here - req := &pipeline.BulkTriggerRequest{ + req := &bean2.BulkTriggerRequest{ CiArtifactId: val.Id, PipelineId: cdPipeline.Id, } @@ -500,7 +502,7 @@ func (impl *DeploymentGroupServiceImpl) TriggerReleaseForDeploymentGroup(trigger } //trigger // apply mapping - _, err = impl.workflowDagExecutor.TriggerBulkDeploymentAsync(requests, triggerRequest.UserId) + _, err = impl.workflowEventPublishService.TriggerBulkDeploymentAsync(requests, triggerRequest.UserId) if err != nil { return nil, err } diff --git a/pkg/eventProcessor/out/WorkflowEventPublishService.go b/pkg/eventProcessor/out/WorkflowEventPublishService.go new file mode 100644 index 0000000000..7ef9f1a6ee --- /dev/null +++ b/pkg/eventProcessor/out/WorkflowEventPublishService.go @@ -0,0 +1,72 @@ +package out + +import ( + "encoding/json" + pubsub "github.com/devtron-labs/common-lib/pubsub-lib" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/pkg/eventProcessor/out/bean" + "github.com/devtron-labs/devtron/pkg/sql" + "go.uber.org/zap" + "time" +) + +type WorkflowEventPublishService interface { + TriggerBulkDeploymentAsync(requests []*bean.BulkTriggerRequest, UserId int32) (interface{}, error) +} + +type WorkflowEventPublishServiceImpl struct { + logger *zap.SugaredLogger + pubSubClient *pubsub.PubSubClientServiceImpl + cdWorkflowRepository pipelineConfig.CdWorkflowRepository +} + +func NewWorkflowEventPublishServiceImpl(logger *zap.SugaredLogger, + pubSubClient *pubsub.PubSubClientServiceImpl, + cdWorkflowRepository pipelineConfig.CdWorkflowRepository) (*WorkflowEventPublishServiceImpl, error) { + impl := &WorkflowEventPublishServiceImpl{ + logger: logger, + pubSubClient: pubSubClient, + cdWorkflowRepository: cdWorkflowRepository, + } + return impl, nil +} + +func (impl *WorkflowEventPublishServiceImpl) TriggerBulkDeploymentAsync(requests []*bean.BulkTriggerRequest, UserId int32) (interface{}, error) { + var cdWorkflows []*pipelineConfig.CdWorkflow + for _, request := range requests { + cdWf := &pipelineConfig.CdWorkflow{ + CiArtifactId: request.CiArtifactId, + PipelineId: request.PipelineId, + AuditLog: sql.AuditLog{CreatedOn: time.Now(), CreatedBy: UserId, UpdatedOn: time.Now(), UpdatedBy: UserId}, + WorkflowStatus: pipelineConfig.REQUEST_ACCEPTED, + } + cdWorkflows = append(cdWorkflows, cdWf) + } + err := impl.cdWorkflowRepository.SaveWorkFlows(cdWorkflows...) + if err != nil { + impl.logger.Errorw("error in saving wfs", "req", requests, "err", err) + return nil, err + } + impl.triggerNatsEventForBulkAction(cdWorkflows) + return nil, nil +} + +func (impl *WorkflowEventPublishServiceImpl) triggerNatsEventForBulkAction(cdWorkflows []*pipelineConfig.CdWorkflow) { + for _, wf := range cdWorkflows { + data, err := json.Marshal(wf) + if err != nil { + wf.WorkflowStatus = pipelineConfig.QUE_ERROR + } else { + err = impl.pubSubClient.Publish(pubsub.BULK_DEPLOY_TOPIC, string(data)) + if err != nil { + wf.WorkflowStatus = pipelineConfig.QUE_ERROR + } else { + wf.WorkflowStatus = pipelineConfig.ENQUEUED + } + } + err = impl.cdWorkflowRepository.UpdateWorkFlow(wf) + if err != nil { + impl.logger.Errorw("error in publishing wf msg", "wf", wf, "err", err) + } + } +} diff --git a/pkg/eventProcessor/out/bean/bean.go b/pkg/eventProcessor/out/bean/bean.go new file mode 100644 index 0000000000..974c99319b --- /dev/null +++ b/pkg/eventProcessor/out/bean/bean.go @@ -0,0 +1,6 @@ +package bean + +type BulkTriggerRequest struct { + CiArtifactId int `sql:"ci_artifact_id"` + PipelineId int `sql:"pipeline_id"` +} diff --git a/pkg/eventProcessor/out/wire_eventProcessorOut.go b/pkg/eventProcessor/out/wire_eventProcessorOut.go new file mode 100644 index 0000000000..2ef126e280 --- /dev/null +++ b/pkg/eventProcessor/out/wire_eventProcessorOut.go @@ -0,0 +1,8 @@ +package out + +import "github.com/google/wire" + +var EventProcessorOutWireSet = wire.NewSet( + NewWorkflowEventPublishServiceImpl, + wire.Bind(new(WorkflowEventPublishService), new(*WorkflowEventPublishServiceImpl)), +) diff --git a/pkg/eventProcessor/wire_eventProcessor.go b/pkg/eventProcessor/wire_eventProcessor.go index 8ed898fb1e..bc05cae450 100644 --- a/pkg/eventProcessor/wire_eventProcessor.go +++ b/pkg/eventProcessor/wire_eventProcessor.go @@ -2,10 +2,12 @@ package eventProcessor import ( "github.com/devtron-labs/devtron/pkg/eventProcessor/in" + "github.com/devtron-labs/devtron/pkg/eventProcessor/out" "github.com/google/wire" ) var EventProcessorWireSet = wire.NewSet( NewCentralEventProcessor, in.EventProcessorInWireSet, + out.EventProcessorOutWireSet, ) diff --git a/pkg/pipeline/AppDeploymentTypeChangeManager.go b/pkg/pipeline/AppDeploymentTypeChangeManager.go index cc2928944e..26aec1e13d 100644 --- a/pkg/pipeline/AppDeploymentTypeChangeManager.go +++ b/pkg/pipeline/AppDeploymentTypeChangeManager.go @@ -32,6 +32,8 @@ import ( chartService "github.com/devtron-labs/devtron/pkg/chart" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" bean3 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" + "github.com/devtron-labs/devtron/pkg/eventProcessor/out" + bean2 "github.com/devtron-labs/devtron/pkg/eventProcessor/out/bean" "github.com/juju/errors" "go.uber.org/zap" "strconv" @@ -60,22 +62,21 @@ type AppDeploymentTypeChangeManager interface { type AppDeploymentTypeChangeManagerImpl struct { logger *zap.SugaredLogger pipelineRepository pipelineConfig.PipelineRepository - workflowDagExecutor WorkflowDagExecutor appService app2.AppService appStatusRepository appStatus.AppStatusRepository helmAppService service.HelmAppService application application2.ServiceClient - appArtifactManager AppArtifactManager - cdPipelineConfigService CdPipelineConfigService - gitOpsConfigReadService config.GitOpsConfigReadService - chartService chartService.ChartService + appArtifactManager AppArtifactManager + cdPipelineConfigService CdPipelineConfigService + gitOpsConfigReadService config.GitOpsConfigReadService + chartService chartService.ChartService + workflowEventPublishService out.WorkflowEventPublishService } func NewAppDeploymentTypeChangeManagerImpl( logger *zap.SugaredLogger, pipelineRepository pipelineConfig.PipelineRepository, - workflowDagExecutor WorkflowDagExecutor, appService app2.AppService, appStatusRepository appStatus.AppStatusRepository, helmAppService service.HelmAppService, @@ -83,19 +84,20 @@ func NewAppDeploymentTypeChangeManagerImpl( appArtifactManager AppArtifactManager, cdPipelineConfigService CdPipelineConfigService, gitOpsConfigReadService config.GitOpsConfigReadService, - chartService chartService.ChartService) *AppDeploymentTypeChangeManagerImpl { + chartService chartService.ChartService, + workflowEventPublishService out.WorkflowEventPublishService) *AppDeploymentTypeChangeManagerImpl { return &AppDeploymentTypeChangeManagerImpl{ - logger: logger, - pipelineRepository: pipelineRepository, - workflowDagExecutor: workflowDagExecutor, - appService: appService, - appStatusRepository: appStatusRepository, - helmAppService: helmAppService, - application: application, - appArtifactManager: appArtifactManager, - cdPipelineConfigService: cdPipelineConfigService, - gitOpsConfigReadService: gitOpsConfigReadService, - chartService: chartService, + logger: logger, + pipelineRepository: pipelineRepository, + appService: appService, + appStatusRepository: appStatusRepository, + helmAppService: helmAppService, + application: application, + appArtifactManager: appArtifactManager, + cdPipelineConfigService: cdPipelineConfigService, + gitOpsConfigReadService: gitOpsConfigReadService, + chartService: chartService, + workflowEventPublishService: workflowEventPublishService, } } @@ -154,7 +156,7 @@ func (impl *AppDeploymentTypeChangeManagerImpl) ChangeDeploymentType(ctx context } // Bulk trigger all the successfully changed pipelines (async) - bulkTriggerRequest := make([]*BulkTriggerRequest, 0) + bulkTriggerRequest := make([]*bean2.BulkTriggerRequest, 0) pipelineIds := make([]int, 0, len(response.SuccessfulPipelines)) for _, item := range response.SuccessfulPipelines { @@ -189,7 +191,7 @@ func (impl *AppDeploymentTypeChangeManagerImpl) ChangeDeploymentType(ctx context continue } - bulkTriggerRequest = append(bulkTriggerRequest, &BulkTriggerRequest{ + bulkTriggerRequest = append(bulkTriggerRequest, &bean2.BulkTriggerRequest{ CiArtifactId: artifactDetails.LatestWfArtifactId, PipelineId: pipeline.Id, }) @@ -205,7 +207,7 @@ func (impl *AppDeploymentTypeChangeManagerImpl) ChangeDeploymentType(ctx context } // Trigger - _, err = impl.workflowDagExecutor.TriggerBulkDeploymentAsync(bulkTriggerRequest, request.UserId) + _, err = impl.workflowEventPublishService.TriggerBulkDeploymentAsync(bulkTriggerRequest, request.UserId) if err != nil { impl.logger.Errorw("failed to bulk trigger cd pipelines with error: "+err.Error(), @@ -331,7 +333,7 @@ func (impl *AppDeploymentTypeChangeManagerImpl) TriggerDeploymentAfterTypeChange successPipelines = append(successPipelines, item.Id) } - bulkTriggerRequest := make([]*BulkTriggerRequest, 0) + bulkTriggerRequest := make([]*bean2.BulkTriggerRequest, 0) pipelineIds := make([]int, 0, len(response.SuccessfulPipelines)) for _, item := range response.SuccessfulPipelines { @@ -365,7 +367,7 @@ func (impl *AppDeploymentTypeChangeManagerImpl) TriggerDeploymentAfterTypeChange continue } - bulkTriggerRequest = append(bulkTriggerRequest, &BulkTriggerRequest{ + bulkTriggerRequest = append(bulkTriggerRequest, &bean2.BulkTriggerRequest{ CiArtifactId: artifactDetails.LatestWfArtifactId, PipelineId: pipeline.Id, }) @@ -379,7 +381,7 @@ func (impl *AppDeploymentTypeChangeManagerImpl) TriggerDeploymentAfterTypeChange return response, nil } - _, err = impl.workflowDagExecutor.TriggerBulkDeploymentAsync(bulkTriggerRequest, request.UserId) + _, err = impl.workflowEventPublishService.TriggerBulkDeploymentAsync(bulkTriggerRequest, request.UserId) if err != nil { impl.logger.Errorw("failed to bulk trigger cd pipelines with error: "+err.Error(), diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index 3d54028992..4755bca229 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -103,7 +103,6 @@ type WorkflowDagExecutor interface { TriggerPreStage(request TriggerRequest) error TriggerDeployment(request TriggerRequest) error ManualCdTrigger(triggerContext TriggerContext, overrideRequest *bean.ValuesOverrideRequest) (int, error) - TriggerBulkDeploymentAsync(requests []*BulkTriggerRequest, UserId int32) (interface{}, error) TriggerStageForBulk(triggerRequest TriggerRequest) error TriggerBulkHibernateAsync(request StopDeploymentGroupRequest, ctx context.Context) (interface{}, error) @@ -2504,35 +2503,6 @@ func (impl *WorkflowDagExecutorImpl) ManualCdTrigger(triggerContext TriggerConte return releaseId, err } -type BulkTriggerRequest struct { - CiArtifactId int `sql:"ci_artifact_id"` - PipelineId int `sql:"pipeline_id"` -} - -func (impl *WorkflowDagExecutorImpl) TriggerBulkDeploymentAsync(requests []*BulkTriggerRequest, UserId int32) (interface{}, error) { - var cdWorkflows []*pipelineConfig.CdWorkflow - for _, request := range requests { - cdWf := &pipelineConfig.CdWorkflow{ - CiArtifactId: request.CiArtifactId, - PipelineId: request.PipelineId, - AuditLog: sql.AuditLog{CreatedOn: time.Now(), CreatedBy: UserId, UpdatedOn: time.Now(), UpdatedBy: UserId}, - WorkflowStatus: pipelineConfig.REQUEST_ACCEPTED, - } - cdWorkflows = append(cdWorkflows, cdWf) - } - err := impl.cdWorkflowRepository.SaveWorkFlows(cdWorkflows...) - if err != nil { - impl.logger.Errorw("error in saving wfs", "req", requests, "err", err) - return nil, err - } - impl.triggerNatsEventForBulkAction(cdWorkflows) - return nil, nil - //return - //publish nats async - //update status - //consume message -} - type DeploymentGroupAppWithEnv struct { EnvironmentId int `json:"environmentId"` DeploymentGroupId int `json:"deploymentGroupId"` @@ -2572,26 +2542,6 @@ func (impl *WorkflowDagExecutorImpl) TriggerBulkHibernateAsync(request StopDeplo return nil, nil } -func (impl *WorkflowDagExecutorImpl) triggerNatsEventForBulkAction(cdWorkflows []*pipelineConfig.CdWorkflow) { - for _, wf := range cdWorkflows { - data, err := json.Marshal(wf) - if err != nil { - wf.WorkflowStatus = pipelineConfig.QUE_ERROR - } else { - err = impl.pubsubClient.Publish(pubsub.BULK_DEPLOY_TOPIC, string(data)) - if err != nil { - wf.WorkflowStatus = pipelineConfig.QUE_ERROR - } else { - wf.WorkflowStatus = pipelineConfig.ENQUEUED - } - } - err = impl.cdWorkflowRepository.UpdateWorkFlow(wf) - if err != nil { - impl.logger.Errorw("error in publishing wf msg", "wf", wf, "err", err) - } - } -} - func extractTimelineFailedStatusDetails(err error) string { errorString := util.GetGRPCErrorDetailedMessage(err) switch errorString { diff --git a/wire_gen.go b/wire_gen.go index 4bb3bcdc70..2e49d4115a 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -118,6 +118,7 @@ import ( "github.com/devtron-labs/devtron/pkg/dockerRegistry" "github.com/devtron-labs/devtron/pkg/eventProcessor" "github.com/devtron-labs/devtron/pkg/eventProcessor/in" + "github.com/devtron-labs/devtron/pkg/eventProcessor/out" "github.com/devtron-labs/devtron/pkg/externalLink" "github.com/devtron-labs/devtron/pkg/generateManifest" "github.com/devtron-labs/devtron/pkg/genericNotes" @@ -468,7 +469,11 @@ func InitializeApp() (*App, error) { triggerServiceImpl := devtronApps.NewTriggerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, argoK8sClientImpl, acdConfig, applicationServiceClientImpl, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, appServiceImpl, helmAppServiceImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl) workflowDagExecutorImpl := pipeline.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, pubSubClientServiceImpl, appServiceImpl, workflowServiceImpl, ciArtifactRepositoryImpl, ciPipelineRepositoryImpl, materialRepositoryImpl, pipelineOverrideRepositoryImpl, userServiceImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, enforcerUtilImpl, eventSimpleFactoryImpl, eventRESTClientImpl, cvePolicyRepositoryImpl, imageScanResultRepositoryImpl, appWorkflowRepositoryImpl, prePostCdScriptHistoryServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineServiceImpl, ciTemplateRepositoryImpl, ciWorkflowRepositoryImpl, appLabelRepositoryImpl, clientImpl, pipelineStageServiceImpl, k8sCommonServiceImpl, globalPluginServiceImpl, pluginInputVariableParserImpl, scopedVariableCMCSManagerImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineStrategyHistoryServiceImpl, appCrudOperationServiceImpl, pipelineConfigRepositoryImpl, dockerRegistryIpsConfigServiceImpl, chartRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, configMapRepositoryImpl, configMapHistoryRepositoryImpl, helmAppServiceImpl, envConfigOverrideRepositoryImpl, mergeUtil, applicationServiceClientImpl, pipelineConfigListenerServiceImpl, customTagServiceImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, imageDigestPolicyServiceImpl, cdWorkflowCommonServiceImpl, triggerServiceImpl) deploymentGroupAppRepositoryImpl := repository.NewDeploymentGroupAppRepositoryImpl(sugaredLogger, db) - deploymentGroupServiceImpl := deploymentGroup.NewDeploymentGroupServiceImpl(appRepositoryImpl, sugaredLogger, pipelineRepositoryImpl, ciPipelineRepositoryImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, deploymentGroupAppRepositoryImpl, ciArtifactRepositoryImpl, appWorkflowRepositoryImpl, workflowDagExecutorImpl) + workflowEventPublishServiceImpl, err := out.NewWorkflowEventPublishServiceImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowRepositoryImpl) + if err != nil { + return nil, err + } + deploymentGroupServiceImpl := deploymentGroup.NewDeploymentGroupServiceImpl(appRepositoryImpl, sugaredLogger, pipelineRepositoryImpl, ciPipelineRepositoryImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, deploymentGroupAppRepositoryImpl, ciArtifactRepositoryImpl, appWorkflowRepositoryImpl, workflowEventPublishServiceImpl) deploymentConfigServiceImpl := pipeline.NewDeploymentConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, pipelineRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, configMapHistoryServiceImpl, scopedVariableCMCSManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) pipelineTriggerRestHandlerImpl := restHandler.NewPipelineRestHandler(appServiceImpl, userServiceImpl, validate, enforcerImpl, teamServiceImpl, sugaredLogger, enforcerUtilImpl, workflowDagExecutorImpl, deploymentGroupServiceImpl, argoUserServiceImpl, deploymentConfigServiceImpl) sseSSE := sse.NewSSE() @@ -512,7 +517,7 @@ func InitializeApp() (*App, error) { globalStrategyMetadataChartRefMappingRepositoryImpl := chartRepoRepository.NewGlobalStrategyMetadataChartRefMappingRepositoryImpl(db, sugaredLogger) devtronAppStrategyServiceImpl := pipeline.NewDevtronAppStrategyServiceImpl(sugaredLogger, chartRepositoryImpl, globalStrategyMetadataChartRefMappingRepositoryImpl, ciCdPipelineOrchestratorImpl, cdPipelineConfigServiceImpl) chartServiceImpl := chart.NewChartServiceImpl(chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, appRepositoryImpl, utilMergeUtil, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, environmentRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) - appDeploymentTypeChangeManagerImpl := pipeline.NewAppDeploymentTypeChangeManagerImpl(sugaredLogger, pipelineRepositoryImpl, workflowDagExecutorImpl, appServiceImpl, appStatusRepositoryImpl, helmAppServiceImpl, applicationServiceClientImpl, appArtifactManagerImpl, cdPipelineConfigServiceImpl, gitOpsConfigReadServiceImpl, chartServiceImpl) + appDeploymentTypeChangeManagerImpl := pipeline.NewAppDeploymentTypeChangeManagerImpl(sugaredLogger, pipelineRepositoryImpl, appServiceImpl, appStatusRepositoryImpl, helmAppServiceImpl, applicationServiceClientImpl, appArtifactManagerImpl, cdPipelineConfigServiceImpl, gitOpsConfigReadServiceImpl, chartServiceImpl, workflowEventPublishServiceImpl) devtronAppConfigServiceImpl := pipeline.NewDevtronAppConfigServiceImpl(sugaredLogger, ciCdPipelineOrchestratorImpl, appRepositoryImpl, pipelineRepositoryImpl, resourceGroupServiceImpl, enforcerUtilImpl, ciMaterialConfigServiceImpl) pipelineBuilderImpl := pipeline.NewPipelineBuilderImpl(sugaredLogger, materialRepositoryImpl, chartRepositoryImpl, ciPipelineConfigServiceImpl, ciMaterialConfigServiceImpl, appArtifactManagerImpl, devtronAppCMCSServiceImpl, devtronAppStrategyServiceImpl, appDeploymentTypeChangeManagerImpl, cdPipelineConfigServiceImpl, devtronAppConfigServiceImpl) deploymentTemplateValidationServiceImpl := deploymentTemplate.NewDeploymentTemplateValidationServiceImpl(sugaredLogger, chartRefServiceImpl, scopedVariableManagerImpl) From be8aa8d86a2eea71e4c81b334c86a88c9dcb0f3b Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Thu, 1 Feb 2024 19:39:57 +0530 Subject: [PATCH 52/64] extracted manifest creation code from WorkflowDagExecutor --- .../argocdServer/ArgoClientWrapperService.go | 39 +- client/argocdServer/adapter/adapter.go | 18 + client/argocdServer/bean/bean.go | 14 +- pkg/app/AppService.go | 134 +- .../manifest/ManifestCreationService.go | 999 +++++++++++++++ pkg/deployment/manifest/bean/bean.go | 25 + .../deployedAppMetrics/wire_appMetrics.go | 16 + .../DeploymentTemplateService.go | 118 +- .../wire_deploymentTemplate.go | 15 + .../manifest/wire_deployment_manifest.go | 18 +- .../trigger/devtronApps/TriggerService.go | 57 +- .../out/WorkflowEventPublishService.go | 126 +- pkg/pipeline/BuildPipelineConfigService.go | 50 +- pkg/pipeline/WorkflowDagExecutor.go | 1123 +---------------- .../DeployedConfigurationHistoryService.go | 47 +- util/GlobalConfig.go | 5 +- wire_gen.go | 61 +- 17 files changed, 1541 insertions(+), 1324 deletions(-) create mode 100644 client/argocdServer/adapter/adapter.go create mode 100644 pkg/deployment/manifest/ManifestCreationService.go create mode 100644 pkg/deployment/manifest/bean/bean.go create mode 100644 pkg/deployment/manifest/deployedAppMetrics/wire_appMetrics.go create mode 100644 pkg/deployment/manifest/deploymentTemplate/wire_deploymentTemplate.go diff --git a/client/argocdServer/ArgoClientWrapperService.go b/client/argocdServer/ArgoClientWrapperService.go index add0a58122..2b2b856009 100644 --- a/client/argocdServer/ArgoClientWrapperService.go +++ b/client/argocdServer/ArgoClientWrapperService.go @@ -2,10 +2,12 @@ package argocdServer import ( "context" + "encoding/json" application2 "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" repository2 "github.com/argoproj/argo-cd/v2/pkg/apiclient/repository" "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" "github.com/caarlos0/env" + "github.com/devtron-labs/devtron/client/argocdServer/adapter" "github.com/devtron-labs/devtron/client/argocdServer/application" "github.com/devtron-labs/devtron/client/argocdServer/bean" "github.com/devtron-labs/devtron/client/argocdServer/repository" @@ -28,17 +30,23 @@ func GetACDDeploymentConfig() (*ACDConfig, error) { type ArgoClientWrapperService interface { - //GetArgoAppWithNormalRefresh - refresh app at argocd side + // GetArgoAppWithNormalRefresh - refresh app at argocd side GetArgoAppWithNormalRefresh(context context.Context, argoAppName string) error - //SyncArgoCDApplicationIfNeededAndRefresh - if ARGO_AUTO_SYNC_ENABLED=true, app will be refreshed to initiate refresh at argoCD side or else it will be synced and refreshed + // SyncArgoCDApplicationIfNeededAndRefresh - if ARGO_AUTO_SYNC_ENABLED=true, app will be refreshed to initiate refresh at argoCD side or else it will be synced and refreshed SyncArgoCDApplicationIfNeededAndRefresh(context context.Context, argoAppName string) error // UpdateArgoCDSyncModeIfNeeded - if ARGO_AUTO_SYNC_ENABLED=true and app is in manual sync mode or vice versa update app UpdateArgoCDSyncModeIfNeeded(ctx context.Context, argoApplication *v1alpha1.Application) (err error) - //RegisterGitOpsRepoInArgo - register a repository in argo-cd + // RegisterGitOpsRepoInArgo - register a repository in argo-cd RegisterGitOpsRepoInArgo(ctx context.Context, repoUrl string) (err error) + + // GetArgoAppByName fetches an argoCd app by its name + GetArgoAppByName(ctx context.Context, appName string) (*v1alpha1.Application, error) + + // PatchArgoCdApp performs a patch operation on an argoCd app + PatchArgoCdApp(ctx context.Context, dto *bean.ArgoCdAppPatchReqDto) error } type ArgoClientWrapperServiceImpl struct { @@ -147,3 +155,28 @@ func (impl *ArgoClientWrapperServiceImpl) RegisterGitOpsRepoInArgo(ctx context.C impl.logger.Infow("gitOps repo registered in argo", "name", repoUrl) return err } + +func (impl *ArgoClientWrapperServiceImpl) GetArgoAppByName(ctx context.Context, appName string) (*v1alpha1.Application, error) { + argoApplication, err := impl.acdClient.Get(ctx, &application2.ApplicationQuery{Name: &appName}) + if err != nil { + impl.logger.Errorw("err in getting argo app by name", "app", appName) + return nil, err + } + return argoApplication, nil +} + +func (impl *ArgoClientWrapperServiceImpl) PatchArgoCdApp(ctx context.Context, dto *bean.ArgoCdAppPatchReqDto) error { + patchReq := adapter.GetArgoCdPatchReqFromDto(dto) + reqbyte, err := json.Marshal(patchReq) + if err != nil { + impl.logger.Errorw("error in creating patch", "err", err) + return err + } + reqString := string(reqbyte) + _, err = impl.acdClient.Patch(ctx, &application2.ApplicationPatchRequest{Patch: &reqString, Name: &dto.ArgoAppName, PatchType: &dto.PatchType}) + if err != nil { + impl.logger.Errorw("error in patching argo pipeline ", "name", dto.ArgoAppName, "patch", reqString, "err", err) + return err + } + return nil +} diff --git a/client/argocdServer/adapter/adapter.go b/client/argocdServer/adapter/adapter.go new file mode 100644 index 0000000000..6dde593c49 --- /dev/null +++ b/client/argocdServer/adapter/adapter.go @@ -0,0 +1,18 @@ +package adapter + +import ( + "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" + "github.com/devtron-labs/devtron/client/argocdServer/bean" +) + +func GetArgoCdPatchReqFromDto(dto *bean.ArgoCdAppPatchReqDto) v1alpha1.Application { + return v1alpha1.Application{ + Spec: v1alpha1.ApplicationSpec{ + Source: &v1alpha1.ApplicationSource{ + Path: dto.ChartLocation, + RepoURL: dto.GitRepoUrl, + TargetRevision: dto.TargetRevision, + }, + }, + } +} diff --git a/client/argocdServer/bean/bean.go b/client/argocdServer/bean/bean.go index 1a75a5c204..8a95483d55 100644 --- a/client/argocdServer/bean/bean.go +++ b/client/argocdServer/bean/bean.go @@ -1,3 +1,15 @@ package bean -const RefreshTypeNormal = "normal" +const ( + RefreshTypeNormal = "normal" + TargetRevisionMaster = "master" + PatchTypeMerge = "merge" +) + +type ArgoCdAppPatchReqDto struct { + ArgoAppName string + ChartLocation string + GitRepoUrl string + TargetRevision string + PatchType string +} diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index 5286557974..e2816cf2d4 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -24,33 +24,20 @@ import ( commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" - bean3 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/bean" bean4 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "io/ioutil" "net/url" - "os" "path" - "path/filepath" "strconv" - "strings" "time" + application2 "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" + "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" "github.com/caarlos0/env" k8sCommonBean "github.com/devtron-labs/common-lib/utils/k8s/commonBean" "github.com/devtron-labs/common-lib/utils/k8s/health" - status2 "github.com/devtron-labs/devtron/pkg/app/status" - repository4 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" - "github.com/devtron-labs/devtron/pkg/resourceQualifiers" - "github.com/devtron-labs/devtron/pkg/variables" - _ "github.com/devtron-labs/devtron/pkg/variables/repository" - "github.com/devtron-labs/devtron/util/argo" - "go.opentelemetry.io/otel" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - chart2 "k8s.io/helm/pkg/proto/hapi/chart" - - application2 "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" - "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/client/argocdServer" "github.com/devtron-labs/devtron/client/argocdServer/application" @@ -60,14 +47,22 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" . "github.com/devtron-labs/devtron/internal/util" + status2 "github.com/devtron-labs/devtron/pkg/app/status" "github.com/devtron-labs/devtron/pkg/appStatus" + repository4 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" "github.com/devtron-labs/devtron/pkg/commonService" + "github.com/devtron-labs/devtron/pkg/resourceQualifiers" "github.com/devtron-labs/devtron/pkg/sql" + "github.com/devtron-labs/devtron/pkg/variables" + _ "github.com/devtron-labs/devtron/pkg/variables/repository" util2 "github.com/devtron-labs/devtron/util" + "github.com/devtron-labs/devtron/util/argo" util "github.com/devtron-labs/devtron/util/event" "github.com/go-pg/pg" + "go.opentelemetry.io/otel" "go.uber.org/zap" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) type AppServiceConfig struct { @@ -123,6 +118,7 @@ type AppServiceImpl struct { chartRefService chartRef.ChartRefService gitOpsConfigReadService config.GitOpsConfigReadService gitOperationService git.GitOperationService + deploymentTemplateService deploymentTemplate.DeploymentTemplateService } type AppService interface { @@ -144,9 +140,6 @@ type AppService interface { CreateGitopsRepo(app *app.App, userId int32) (gitopsRepoName string, chartGitAttr *commonBean.ChartGitAttribute, err error) GetDeployedManifestByPipelineIdAndCDWorkflowId(appId int, envId int, cdWorkflowId int, ctx context.Context) ([]byte, error) //SetPipelineFieldsInOverrideRequest(overrideRequest *bean.ValuesOverrideRequest, pipeline *pipelineConfig.Pipeline) - - BuildChartAndGetPath(appName string, envOverride *chartConfig.EnvConfigOverride, ctx context.Context) (string, error) - IsDevtronAsyncInstallModeEnabled(deploymentAppType string) bool } func NewAppService( @@ -176,7 +169,8 @@ func NewAppService( scopedVariableManager variables.ScopedVariableCMCSManager, acdConfig *argocdServer.ACDConfig, chartRefService chartRef.ChartRefService, gitOpsConfigReadService config.GitOpsConfigReadService, - gitOperationService git.GitOperationService) *AppServiceImpl { + gitOperationService git.GitOperationService, + deploymentTemplateService deploymentTemplate.DeploymentTemplateService) *AppServiceImpl { appServiceImpl := &AppServiceImpl{ environmentConfigRepository: environmentConfigRepository, mergeUtil: mergeUtil, @@ -207,6 +201,7 @@ func NewAppService( chartRefService: chartRefService, gitOpsConfigReadService: gitOpsConfigReadService, gitOperationService: gitOperationService, + deploymentTemplateService: deploymentTemplateService, } return appServiceImpl } @@ -839,7 +834,7 @@ func (impl *AppServiceImpl) GetDeployedManifestByPipelineIdAndCDWorkflowId(appId } appName := pipeline[0].App.AppName - builtChartPath, err := impl.BuildChartAndGetPath(appName, envConfigOverride, ctx) + builtChartPath, err := impl.deploymentTemplateService.BuildChartAndGetPath(appName, envConfigOverride, ctx) if err != nil { impl.logger.Errorw("error in parsing reference chart", "err", err) return manifestByteArray, err @@ -862,48 +857,6 @@ func (impl *AppServiceImpl) GetDeployedManifestByPipelineIdAndCDWorkflowId(appId } -func (impl *AppServiceImpl) BuildChartAndGetPath(appName string, envOverride *chartConfig.EnvConfigOverride, ctx context.Context) (string, error) { - - if !strings.HasSuffix(envOverride.Chart.ChartLocation, fmt.Sprintf("%s%s", "/", envOverride.Chart.ChartVersion)) { - _, span := otel.Tracer("orchestrator").Start(ctx, "autoHealChartLocationInChart") - err := impl.autoHealChartLocationInChart(ctx, envOverride) - span.End() - if err != nil { - return "", err - } - } - chartMetaData := &chart2.Metadata{ - Name: appName, - Version: envOverride.Chart.ChartVersion, - } - referenceTemplatePath := path.Join(bean3.RefChartDirPath, envOverride.Chart.ReferenceTemplate) - // Load custom charts to referenceTemplatePath if not exists - if _, err := os.Stat(referenceTemplatePath); os.IsNotExist(err) { - chartRefValue, err := impl.chartRefService.FindById(envOverride.Chart.ChartRefId) - if err != nil { - impl.logger.Errorw("error in fetching ChartRef data", "err", err) - return "", err - } - if chartRefValue.ChartData != nil { - chartInfo, err := impl.chartRefService.ExtractChartIfMissing(chartRefValue.ChartData, bean3.RefChartDirPath, chartRefValue.Location) - if chartInfo != nil && chartInfo.TemporaryFolder != "" { - err1 := os.RemoveAll(chartInfo.TemporaryFolder) - if err1 != nil { - impl.logger.Errorw("error in deleting temp dir ", "err", err) - } - } - return "", err - } - } - _, span := otel.Tracer("orchestrator").Start(ctx, "chartTemplateService.BuildChart") - tempReferenceTemplateDir, err := impl.chartTemplateService.BuildChart(ctx, chartMetaData, referenceTemplatePath) - span.End() - if err != nil { - return "", err - } - return tempReferenceTemplateDir, nil -} - func (impl *AppServiceImpl) CreateGitopsRepo(app *app.App, userId int32) (gitopsRepoName string, chartGitAttr *commonBean.ChartGitAttribute, err error) { chart, err := impl.chartRepository.FindLatestChartForAppByAppId(app.Id) if err != nil && pg.ErrNoRows != err { @@ -940,49 +893,6 @@ func (impl *AppServiceImpl) saveTimeline(overrideRequest *bean.ValuesOverrideReq } } -func (impl *AppServiceImpl) autoHealChartLocationInChart(ctx context.Context, envOverride *chartConfig.EnvConfigOverride) error { - chartId := envOverride.Chart.Id - impl.logger.Infow("auto-healing: Chart location in chart not correct. modifying ", "chartId", chartId, - "current chartLocation", envOverride.Chart.ChartLocation, "current chartVersion", envOverride.Chart.ChartVersion) - - // get chart from DB (getting it from DB because envOverride.Chart does not have full row of DB) - _, span := otel.Tracer("orchestrator").Start(ctx, "chartRepository.FindById") - chart, err := impl.chartRepository.FindById(chartId) - span.End() - if err != nil { - impl.logger.Errorw("error occurred while fetching chart from DB", "chartId", chartId, "err", err) - return err - } - - // get chart ref from DB (to get location) - chartRefId := chart.ChartRefId - _, span = otel.Tracer("orchestrator").Start(ctx, "chartRefRepository.FindById") - chartRefDto, err := impl.chartRefService.FindById(chartRefId) - span.End() - if err != nil { - impl.logger.Errorw("error occurred while fetching chartRef from DB", "chartRefId", chartRefId, "err", err) - return err - } - - // build new chart location - newChartLocation := filepath.Join(chartRefDto.Location, envOverride.Chart.ChartVersion) - impl.logger.Infow("new chart location build", "chartId", chartId, "newChartLocation", newChartLocation) - - // update chart in DB - chart.ChartLocation = newChartLocation - _, span = otel.Tracer("orchestrator").Start(ctx, "chartRepository.Update") - err = impl.chartRepository.Update(chart) - span.End() - if err != nil { - impl.logger.Errorw("error occurred while saving chart into DB", "chartId", chartId, "err", err) - return err - } - - // update newChartLocation in model - envOverride.Chart.ChartLocation = newChartLocation - return nil -} - // FIXME tmp workaround func (impl *AppServiceImpl) GetCmSecretNew(appId int, envId int, isJob bool, scope resourceQualifiers.Scope) (*bean.ConfigMapJson, *bean.ConfigSecretJson, error) { var configMapJson string @@ -1236,15 +1146,3 @@ func (impl *AppServiceImpl) UpdateCdWorkflowRunnerByACDObject(app *v1alpha1.Appl util2.TriggerCDMetrics(cdMetrics, impl.appStatusConfig.ExposeCDMetrics) return nil } - -const kedaAutoscaling = "kedaAutoscaling" -const HorizontalPodAutoscaler = "HorizontalPodAutoscaler" -const fullnameOverride = "fullnameOverride" -const nameOverride = "nameOverride" -const enabled = "enabled" -const replicaCount = "replicaCount" - -func (impl *AppServiceImpl) IsDevtronAsyncInstallModeEnabled(deploymentAppType string) bool { - return impl.appStatusConfig.EnableAsyncInstallDevtronChart && - deploymentAppType == bean4.Helm -} diff --git a/pkg/deployment/manifest/ManifestCreationService.go b/pkg/deployment/manifest/ManifestCreationService.go new file mode 100644 index 0000000000..fece14b7f4 --- /dev/null +++ b/pkg/deployment/manifest/ManifestCreationService.go @@ -0,0 +1,999 @@ +package manifest + +import ( + "context" + "encoding/json" + "fmt" + application3 "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" + "github.com/aws/aws-sdk-go/service/autoscaling" + util5 "github.com/devtron-labs/common-lib/utils/k8s" + "github.com/devtron-labs/devtron/api/bean" + application2 "github.com/devtron-labs/devtron/client/argocdServer/application" + "github.com/devtron-labs/devtron/internal/middleware" + "github.com/devtron-labs/devtron/internal/sql/models" + "github.com/devtron-labs/devtron/internal/sql/repository" + "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/app" + bean2 "github.com/devtron-labs/devtron/pkg/bean" + chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" + repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" + bean3 "github.com/devtron-labs/devtron/pkg/deployment/manifest/bean" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" + "github.com/devtron-labs/devtron/pkg/dockerRegistry" + "github.com/devtron-labs/devtron/pkg/imageDigestPolicy" + "github.com/devtron-labs/devtron/pkg/k8s" + repository3 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" + "github.com/devtron-labs/devtron/pkg/resourceQualifiers" + "github.com/devtron-labs/devtron/pkg/sql" + util3 "github.com/devtron-labs/devtron/pkg/util" + "github.com/devtron-labs/devtron/pkg/variables" + "github.com/devtron-labs/devtron/pkg/variables/parsers" + repository5 "github.com/devtron-labs/devtron/pkg/variables/repository" + util4 "github.com/devtron-labs/devtron/util" + "github.com/go-pg/pg" + errors2 "github.com/juju/errors" + "github.com/pkg/errors" + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" + "go.opentelemetry.io/otel" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/runtime/schema" + "strconv" + "strings" + "time" +) + +type ManifestCreationService interface { + GetValuesOverrideForTrigger(overrideRequest *bean.ValuesOverrideRequest, triggeredAt time.Time, ctx context.Context) (*app.ValuesOverrideResponse, error) +} + +type ManifestCreationServiceImpl struct { + logger *zap.SugaredLogger + dockerRegistryIpsConfigService dockerRegistry.DockerRegistryIpsConfigService + chartRefService chartRef.ChartRefService + scopedVariableManager variables.ScopedVariableCMCSManager + k8sCommonService k8s.K8sCommonService + deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService + imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService + mergeUtil *util.MergeUtil + appCrudOperationService app.AppCrudOperationService + + acdClient application2.ServiceClient //TODO: replace with argoClientWrapperService + + configMapHistoryRepository repository3.ConfigMapHistoryRepository + configMapRepository chartConfig.ConfigMapRepository + chartRepository chartRepoRepository.ChartRepository + environmentConfigRepository chartConfig.EnvConfigOverrideRepository + envRepository repository2.EnvironmentRepository + pipelineRepository pipelineConfig.PipelineRepository + ciArtifactRepository repository.CiArtifactRepository + pipelineOverrideRepository chartConfig.PipelineOverrideRepository + strategyHistoryRepository repository3.PipelineStrategyHistoryRepository + pipelineConfigRepository chartConfig.PipelineConfigRepository + deploymentTemplateHistoryRepository repository3.DeploymentTemplateHistoryRepository +} + +func NewManifestCreationServiceImpl(logger *zap.SugaredLogger, + dockerRegistryIpsConfigService dockerRegistry.DockerRegistryIpsConfigService, + chartRefService chartRef.ChartRefService, + scopedVariableManager variables.ScopedVariableCMCSManager, + k8sCommonService k8s.K8sCommonService, + deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService, + imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService, + mergeUtil *util.MergeUtil, + appCrudOperationService app.AppCrudOperationService, + acdClient application2.ServiceClient, + configMapHistoryRepository repository3.ConfigMapHistoryRepository, + configMapRepository chartConfig.ConfigMapRepository, + chartRepository chartRepoRepository.ChartRepository, + environmentConfigRepository chartConfig.EnvConfigOverrideRepository, + envRepository repository2.EnvironmentRepository, + pipelineRepository pipelineConfig.PipelineRepository, + ciArtifactRepository repository.CiArtifactRepository, + pipelineOverrideRepository chartConfig.PipelineOverrideRepository, + strategyHistoryRepository repository3.PipelineStrategyHistoryRepository, + pipelineConfigRepository chartConfig.PipelineConfigRepository, + deploymentTemplateHistoryRepository repository3.DeploymentTemplateHistoryRepository) *ManifestCreationServiceImpl { + return &ManifestCreationServiceImpl{ + logger: logger, + dockerRegistryIpsConfigService: dockerRegistryIpsConfigService, + chartRefService: chartRefService, + scopedVariableManager: scopedVariableManager, + k8sCommonService: k8sCommonService, + deployedAppMetricsService: deployedAppMetricsService, + imageDigestPolicyService: imageDigestPolicyService, + mergeUtil: mergeUtil, + appCrudOperationService: appCrudOperationService, + configMapRepository: configMapRepository, + acdClient: acdClient, + configMapHistoryRepository: configMapHistoryRepository, + chartRepository: chartRepository, + environmentConfigRepository: environmentConfigRepository, + envRepository: envRepository, + pipelineRepository: pipelineRepository, + ciArtifactRepository: ciArtifactRepository, + pipelineOverrideRepository: pipelineOverrideRepository, + strategyHistoryRepository: strategyHistoryRepository, + pipelineConfigRepository: pipelineConfigRepository, + deploymentTemplateHistoryRepository: deploymentTemplateHistoryRepository, + } +} + +func (impl *ManifestCreationServiceImpl) GetValuesOverrideForTrigger(overrideRequest *bean.ValuesOverrideRequest, triggeredAt time.Time, ctx context.Context) (*app.ValuesOverrideResponse, error) { + if overrideRequest.DeploymentType == models.DEPLOYMENTTYPE_UNKNOWN { + overrideRequest.DeploymentType = models.DEPLOYMENTTYPE_DEPLOY + } + if len(overrideRequest.DeploymentWithConfig) == 0 { + overrideRequest.DeploymentWithConfig = bean.DEPLOYMENT_CONFIG_TYPE_LAST_SAVED + } + valuesOverrideResponse := &app.ValuesOverrideResponse{} + isPipelineOverrideCreated := overrideRequest.PipelineOverrideId > 0 + pipeline, err := impl.pipelineRepository.FindById(overrideRequest.PipelineId) + valuesOverrideResponse.Pipeline = pipeline + if err != nil { + impl.logger.Errorw("error in fetching pipeline by pipeline id", "err", err, "pipeline-id-", overrideRequest.PipelineId) + return valuesOverrideResponse, err + } + + _, span := otel.Tracer("orchestrator").Start(ctx, "ciArtifactRepository.Get") + artifact, err := impl.ciArtifactRepository.Get(overrideRequest.CiArtifactId) + valuesOverrideResponse.Artifact = artifact + span.End() + if err != nil { + return valuesOverrideResponse, err + } + overrideRequest.Image = artifact.Image + + strategy, err := impl.getDeploymentStrategyByTriggerType(overrideRequest, ctx) + valuesOverrideResponse.PipelineStrategy = strategy + if err != nil { + impl.logger.Errorw("error in getting strategy by trigger type", "err", err) + return valuesOverrideResponse, err + } + + envOverride, err := impl.getEnvOverrideByTriggerType(overrideRequest, triggeredAt, ctx) + valuesOverrideResponse.EnvOverride = envOverride + if err != nil { + impl.logger.Errorw("error in getting env override by trigger type", "err", err) + return valuesOverrideResponse, err + } + appMetrics, err := impl.getAppMetricsByTriggerType(overrideRequest, ctx) + valuesOverrideResponse.AppMetrics = appMetrics + if err != nil { + impl.logger.Errorw("error in getting app metrics by trigger type", "err", err) + return valuesOverrideResponse, err + } + var ( + pipelineOverride *chartConfig.PipelineOverride + configMapJson, appLabelJsonByte []byte + ) + + // Conditional Block based on PipelineOverrideCreated --> start + if !isPipelineOverrideCreated { + _, span = otel.Tracer("orchestrator").Start(ctx, "savePipelineOverride") + pipelineOverride, err = impl.savePipelineOverride(overrideRequest, envOverride.Id, triggeredAt) + span.End() + if err != nil { + return valuesOverrideResponse, err + } + overrideRequest.PipelineOverrideId = pipelineOverride.Id + } else { + pipelineOverride, err = impl.pipelineOverrideRepository.FindById(overrideRequest.PipelineOverrideId) + if err != nil { + impl.logger.Errorw("error in getting pipelineOverride for valuesOverrideResponse", "PipelineOverrideId", overrideRequest.PipelineOverrideId) + return nil, err + } + } + // Conditional Block based on PipelineOverrideCreated --> end + valuesOverrideResponse.PipelineOverride = pipelineOverride + + //TODO: check status and apply lock + releaseOverrideJson, err := impl.getReleaseOverride(envOverride, overrideRequest, artifact, pipelineOverride, strategy, &appMetrics) + valuesOverrideResponse.ReleaseOverrideJSON = releaseOverrideJson + if err != nil { + return valuesOverrideResponse, err + } + + // Conditional Block based on PipelineOverrideCreated --> start + if !isPipelineOverrideCreated { + chartVersion := envOverride.Chart.ChartVersion + _, span = otel.Tracer("orchestrator").Start(ctx, "getConfigMapAndSecretJsonV2") + scope := getScopeForVariables(overrideRequest, envOverride) + request := createConfigMapAndSecretJsonRequest(overrideRequest, envOverride, chartVersion, scope) + + configMapJson, err = impl.getConfigMapAndSecretJsonV2(request, envOverride) + span.End() + if err != nil { + impl.logger.Errorw("error in fetching config map n secret ", "err", err) + configMapJson = nil + } + _, span = otel.Tracer("orchestrator").Start(ctx, "appCrudOperationService.GetLabelsByAppIdForDeployment") + appLabelJsonByte, err = impl.appCrudOperationService.GetLabelsByAppIdForDeployment(overrideRequest.AppId) + span.End() + if err != nil { + impl.logger.Errorw("error in fetching app labels for gitOps commit", "err", err) + appLabelJsonByte = nil + } + + mergedValues, err := impl.mergeOverrideValues(envOverride, releaseOverrideJson, configMapJson, appLabelJsonByte, strategy) + appName := fmt.Sprintf("%s-%s", overrideRequest.AppName, envOverride.Environment.Name) + mergedValues = impl.autoscalingCheckBeforeTrigger(ctx, appName, envOverride.Namespace, mergedValues, overrideRequest) + + _, span = otel.Tracer("orchestrator").Start(ctx, "dockerRegistryIpsConfigService.HandleImagePullSecretOnApplicationDeployment") + // handle image pull secret if access given + mergedValues, err = impl.dockerRegistryIpsConfigService.HandleImagePullSecretOnApplicationDeployment(envOverride.Environment, artifact, pipeline.CiPipelineId, mergedValues) + span.End() + if err != nil { + return valuesOverrideResponse, err + } + + pipelineOverride.PipelineMergedValues = string(mergedValues) + valuesOverrideResponse.MergedValues = string(mergedValues) + err = impl.pipelineOverrideRepository.Update(pipelineOverride) + if err != nil { + return valuesOverrideResponse, err + } + valuesOverrideResponse.PipelineOverride = pipelineOverride + } else { + valuesOverrideResponse.MergedValues = pipelineOverride.PipelineMergedValues + } + // Conditional Block based on PipelineOverrideCreated --> end + return valuesOverrideResponse, err +} + +func (impl *ManifestCreationServiceImpl) getDeploymentStrategyByTriggerType(overrideRequest *bean.ValuesOverrideRequest, ctx context.Context) (*chartConfig.PipelineStrategy, error) { + + strategy := &chartConfig.PipelineStrategy{} + var err error + if overrideRequest.DeploymentWithConfig == bean.DEPLOYMENT_CONFIG_TYPE_SPECIFIC_TRIGGER { + _, span := otel.Tracer("orchestrator").Start(ctx, "strategyHistoryRepository.GetHistoryByPipelineIdAndWfrId") + strategyHistory, err := impl.strategyHistoryRepository.GetHistoryByPipelineIdAndWfrId(overrideRequest.PipelineId, overrideRequest.WfrIdForDeploymentWithSpecificTrigger) + span.End() + if err != nil { + impl.logger.Errorw("error in getting deployed strategy history by pipelineId and wfrId", "err", err, "pipelineId", overrideRequest.PipelineId, "wfrId", overrideRequest.WfrIdForDeploymentWithSpecificTrigger) + return nil, err + } + strategy.Strategy = strategyHistory.Strategy + strategy.Config = strategyHistory.Config + strategy.PipelineId = overrideRequest.PipelineId + } else if overrideRequest.DeploymentWithConfig == bean.DEPLOYMENT_CONFIG_TYPE_LAST_SAVED { + if overrideRequest.ForceTrigger { + _, span := otel.Tracer("orchestrator").Start(ctx, "pipelineConfigRepository.GetDefaultStrategyByPipelineId") + strategy, err = impl.pipelineConfigRepository.GetDefaultStrategyByPipelineId(overrideRequest.PipelineId) + span.End() + } else { + var deploymentTemplate chartRepoRepository.DeploymentStrategy + if overrideRequest.DeploymentTemplate == "ROLLING" { + deploymentTemplate = chartRepoRepository.DEPLOYMENT_STRATEGY_ROLLING + } else if overrideRequest.DeploymentTemplate == "BLUE-GREEN" { + deploymentTemplate = chartRepoRepository.DEPLOYMENT_STRATEGY_BLUE_GREEN + } else if overrideRequest.DeploymentTemplate == "CANARY" { + deploymentTemplate = chartRepoRepository.DEPLOYMENT_STRATEGY_CANARY + } else if overrideRequest.DeploymentTemplate == "RECREATE" { + deploymentTemplate = chartRepoRepository.DEPLOYMENT_STRATEGY_RECREATE + } + + if len(deploymentTemplate) > 0 { + _, span := otel.Tracer("orchestrator").Start(ctx, "pipelineConfigRepository.FindByStrategyAndPipelineId") + strategy, err = impl.pipelineConfigRepository.FindByStrategyAndPipelineId(deploymentTemplate, overrideRequest.PipelineId) + span.End() + } else { + _, span := otel.Tracer("orchestrator").Start(ctx, "pipelineConfigRepository.GetDefaultStrategyByPipelineId") + strategy, err = impl.pipelineConfigRepository.GetDefaultStrategyByPipelineId(overrideRequest.PipelineId) + span.End() + } + } + if err != nil && errors2.IsNotFound(err) == false { + impl.logger.Errorf("invalid state", "err", err, "req", strategy) + return nil, err + } + } + return strategy, nil +} + +func (impl *ManifestCreationServiceImpl) getEnvOverrideByTriggerType(overrideRequest *bean.ValuesOverrideRequest, triggeredAt time.Time, ctx context.Context) (*chartConfig.EnvConfigOverride, error) { + + envOverride := &chartConfig.EnvConfigOverride{} + + var err error + if overrideRequest.DeploymentWithConfig == bean.DEPLOYMENT_CONFIG_TYPE_SPECIFIC_TRIGGER { + _, span := otel.Tracer("orchestrator").Start(ctx, "deploymentTemplateHistoryRepository.GetHistoryByPipelineIdAndWfrId") + deploymentTemplateHistory, err := impl.deploymentTemplateHistoryRepository.GetHistoryByPipelineIdAndWfrId(overrideRequest.PipelineId, overrideRequest.WfrIdForDeploymentWithSpecificTrigger) + //VARIABLE_SNAPSHOT_GET and resolve + + span.End() + if err != nil { + impl.logger.Errorw("error in getting deployed deployment template history by pipelineId and wfrId", "err", err, "pipelineId", &overrideRequest, "wfrId", overrideRequest.WfrIdForDeploymentWithSpecificTrigger) + return nil, err + } + templateName := deploymentTemplateHistory.TemplateName + templateVersion := deploymentTemplateHistory.TemplateVersion + if templateName == "Rollout Deployment" { + templateName = "" + } + //getting chart_ref by id + _, span = otel.Tracer("orchestrator").Start(ctx, "chartRefRepository.FindByVersionAndName") + chartRefDto, err := impl.chartRefService.FindByVersionAndName(templateName, templateVersion) + span.End() + if err != nil { + impl.logger.Errorw("error in getting chartRef by version and name", "err", err, "version", templateVersion, "name", templateName) + return nil, err + } + //assuming that if a chartVersion is deployed then it's envConfigOverride will be available + _, span = otel.Tracer("orchestrator").Start(ctx, "environmentConfigRepository.GetByAppIdEnvIdAndChartRefId") + envOverride, err = impl.environmentConfigRepository.GetByAppIdEnvIdAndChartRefId(overrideRequest.AppId, overrideRequest.EnvId, chartRefDto.Id) + span.End() + if err != nil { + impl.logger.Errorw("error in getting envConfigOverride for pipeline for specific chartVersion", "err", err, "appId", overrideRequest.AppId, "envId", overrideRequest.EnvId, "chartRefId", chartRefDto.Id) + return nil, err + } + + _, span = otel.Tracer("orchestrator").Start(ctx, "envRepository.FindById") + env, err := impl.envRepository.FindById(envOverride.TargetEnvironment) + span.End() + if err != nil { + impl.logger.Errorw("unable to find env", "err", err) + return nil, err + } + envOverride.Environment = env + + //updating historical data in envConfigOverride and appMetrics flag + envOverride.IsOverride = true + envOverride.EnvOverrideValues = deploymentTemplateHistory.Template + reference := repository5.HistoryReference{ + HistoryReferenceId: deploymentTemplateHistory.Id, + HistoryReferenceType: repository5.HistoryReferenceTypeDeploymentTemplate, + } + variableMap, resolvedTemplate, err := impl.scopedVariableManager.GetVariableSnapshotAndResolveTemplate(envOverride.EnvOverrideValues, parsers.JsonVariableTemplate, reference, true, false) + envOverride.ResolvedEnvOverrideValues = resolvedTemplate + envOverride.VariableSnapshot = variableMap + if err != nil { + return envOverride, err + } + } else if overrideRequest.DeploymentWithConfig == bean.DEPLOYMENT_CONFIG_TYPE_LAST_SAVED { + _, span := otel.Tracer("orchestrator").Start(ctx, "environmentConfigRepository.ActiveEnvConfigOverride") + envOverride, err = impl.environmentConfigRepository.ActiveEnvConfigOverride(overrideRequest.AppId, overrideRequest.EnvId) + + var chart *chartRepoRepository.Chart + span.End() + if err != nil { + impl.logger.Errorw("invalid state", "err", err, "req", overrideRequest) + return nil, err + } + if envOverride.Id == 0 { + _, span = otel.Tracer("orchestrator").Start(ctx, "chartRepository.FindLatestChartForAppByAppId") + chart, err = impl.chartRepository.FindLatestChartForAppByAppId(overrideRequest.AppId) + span.End() + if err != nil { + impl.logger.Errorw("invalid state", "err", err, "req", overrideRequest) + return nil, err + } + _, span = otel.Tracer("orchestrator").Start(ctx, "environmentConfigRepository.FindChartByAppIdAndEnvIdAndChartRefId") + envOverride, err = impl.environmentConfigRepository.FindChartByAppIdAndEnvIdAndChartRefId(overrideRequest.AppId, overrideRequest.EnvId, chart.ChartRefId) + span.End() + if err != nil && !errors2.IsNotFound(err) { + impl.logger.Errorw("invalid state", "err", err, "req", overrideRequest) + return nil, err + } + + //creating new env override config + if errors2.IsNotFound(err) || envOverride == nil { + _, span = otel.Tracer("orchestrator").Start(ctx, "envRepository.FindById") + environment, err := impl.envRepository.FindById(overrideRequest.EnvId) + span.End() + if err != nil && !util.IsErrNoRows(err) { + return nil, err + } + envOverride = &chartConfig.EnvConfigOverride{ + Active: true, + ManualReviewed: true, + Status: models.CHARTSTATUS_SUCCESS, + TargetEnvironment: overrideRequest.EnvId, + ChartId: chart.Id, + AuditLog: sql.AuditLog{UpdatedBy: overrideRequest.UserId, UpdatedOn: triggeredAt, CreatedOn: triggeredAt, CreatedBy: overrideRequest.UserId}, + Namespace: environment.Namespace, + IsOverride: false, + EnvOverrideValues: "{}", + Latest: false, + IsBasicViewLocked: chart.IsBasicViewLocked, + CurrentViewEditor: chart.CurrentViewEditor, + } + _, span = otel.Tracer("orchestrator").Start(ctx, "environmentConfigRepository.Save") + err = impl.environmentConfigRepository.Save(envOverride) + span.End() + if err != nil { + impl.logger.Errorw("error in creating envConfig", "data", envOverride, "error", err) + return nil, err + } + } + envOverride.Chart = chart + } else if envOverride.Id > 0 && !envOverride.IsOverride { + _, span = otel.Tracer("orchestrator").Start(ctx, "chartRepository.FindLatestChartForAppByAppId") + chart, err = impl.chartRepository.FindLatestChartForAppByAppId(overrideRequest.AppId) + span.End() + if err != nil { + impl.logger.Errorw("invalid state", "err", err, "req", overrideRequest) + return nil, err + } + envOverride.Chart = chart + } + + _, span = otel.Tracer("orchestrator").Start(ctx, "envRepository.FindById") + env, err := impl.envRepository.FindById(envOverride.TargetEnvironment) + span.End() + if err != nil { + impl.logger.Errorw("unable to find env", "err", err) + return nil, err + } + envOverride.Environment = env + scope := getScopeForVariables(overrideRequest, envOverride) + if envOverride.IsOverride { + + entity := repository5.GetEntity(envOverride.Id, repository5.EntityTypeDeploymentTemplateEnvLevel) + resolvedTemplate, variableMap, err := impl.scopedVariableManager.GetMappedVariablesAndResolveTemplate(envOverride.EnvOverrideValues, scope, entity, true) + envOverride.ResolvedEnvOverrideValues = resolvedTemplate + envOverride.VariableSnapshot = variableMap + if err != nil { + return envOverride, err + } + + } else { + entity := repository5.GetEntity(chart.Id, repository5.EntityTypeDeploymentTemplateAppLevel) + resolvedTemplate, variableMap, err := impl.scopedVariableManager.GetMappedVariablesAndResolveTemplate(chart.GlobalOverride, scope, entity, true) + envOverride.Chart.ResolvedGlobalOverride = resolvedTemplate + envOverride.VariableSnapshot = variableMap + if err != nil { + return envOverride, err + } + + } + } + + return envOverride, nil +} + +func (impl *ManifestCreationServiceImpl) getAppMetricsByTriggerType(overrideRequest *bean.ValuesOverrideRequest, ctx context.Context) (bool, error) { + + var appMetrics bool + if overrideRequest.DeploymentWithConfig == bean.DEPLOYMENT_CONFIG_TYPE_SPECIFIC_TRIGGER { + _, span := otel.Tracer("orchestrator").Start(ctx, "deploymentTemplateHistoryRepository.GetHistoryByPipelineIdAndWfrId") + deploymentTemplateHistory, err := impl.deploymentTemplateHistoryRepository.GetHistoryByPipelineIdAndWfrId(overrideRequest.PipelineId, overrideRequest.WfrIdForDeploymentWithSpecificTrigger) + span.End() + if err != nil { + impl.logger.Errorw("error in getting deployed deployment template history by pipelineId and wfrId", "err", err, "pipelineId", &overrideRequest, "wfrId", overrideRequest.WfrIdForDeploymentWithSpecificTrigger) + return appMetrics, err + } + appMetrics = deploymentTemplateHistory.IsAppMetricsEnabled + + } else if overrideRequest.DeploymentWithConfig == bean.DEPLOYMENT_CONFIG_TYPE_LAST_SAVED { + _, span := otel.Tracer("orchestrator").Start(ctx, "deployedAppMetricsService.GetMetricsFlagForAPipelineByAppIdAndEnvId") + isAppMetricsEnabled, err := impl.deployedAppMetricsService.GetMetricsFlagForAPipelineByAppIdAndEnvId(overrideRequest.AppId, overrideRequest.EnvId) + if err != nil { + impl.logger.Errorw("error, GetMetricsFlagForAPipelineByAppIdAndEnvId", "err", err, "appId", overrideRequest.AppId, "envId", overrideRequest.EnvId) + return appMetrics, err + } + span.End() + appMetrics = isAppMetricsEnabled + } + return appMetrics, nil +} + +func (impl *ManifestCreationServiceImpl) mergeOverrideValues(envOverride *chartConfig.EnvConfigOverride, + releaseOverrideJson string, + configMapJson []byte, + appLabelJsonByte []byte, + strategy *chartConfig.PipelineStrategy, +) (mergedValues []byte, err error) { + + //merge three values on the fly + //ordering is important here + //global < environment < db< release + var merged []byte + if !envOverride.IsOverride { + merged, err = impl.mergeUtil.JsonPatch([]byte("{}"), []byte(envOverride.Chart.ResolvedGlobalOverride)) + if err != nil { + return nil, err + } + } else { + merged, err = impl.mergeUtil.JsonPatch([]byte("{}"), []byte(envOverride.ResolvedEnvOverrideValues)) + if err != nil { + return nil, err + } + } + if strategy != nil && len(strategy.Config) > 0 { + merged, err = impl.mergeUtil.JsonPatch(merged, []byte(strategy.Config)) + if err != nil { + return nil, err + } + } + merged, err = impl.mergeUtil.JsonPatch(merged, []byte(releaseOverrideJson)) + if err != nil { + return nil, err + } + if configMapJson != nil { + merged, err = impl.mergeUtil.JsonPatch(merged, configMapJson) + if err != nil { + return nil, err + } + } + if appLabelJsonByte != nil { + merged, err = impl.mergeUtil.JsonPatch(merged, appLabelJsonByte) + if err != nil { + return nil, err + } + } + return merged, nil +} + +func (impl *ManifestCreationServiceImpl) getConfigMapAndSecretJsonV2(request bean3.ConfigMapAndSecretJsonV2, envOverride *chartConfig.EnvConfigOverride) ([]byte, error) { + + var configMapJson, secretDataJson, configMapJsonApp, secretDataJsonApp, configMapJsonEnv, secretDataJsonEnv string + + var err error + configMapA := &chartConfig.ConfigMapAppModel{} + configMapE := &chartConfig.ConfigMapEnvModel{} + configMapHistory, secretHistory := &repository3.ConfigmapAndSecretHistory{}, &repository3.ConfigmapAndSecretHistory{} + + merged := []byte("{}") + if request.DeploymentWithConfig == bean.DEPLOYMENT_CONFIG_TYPE_LAST_SAVED { + configMapA, err = impl.configMapRepository.GetByAppIdAppLevel(request.AppId) + if err != nil && pg.ErrNoRows != err { + return []byte("{}"), err + } + if configMapA != nil && configMapA.Id > 0 { + configMapJsonApp = configMapA.ConfigMapData + secretDataJsonApp = configMapA.SecretData + } + + configMapE, err = impl.configMapRepository.GetByAppIdAndEnvIdEnvLevel(request.AppId, request.EnvId) + if err != nil && pg.ErrNoRows != err { + return []byte("{}"), err + } + if configMapE != nil && configMapE.Id > 0 { + configMapJsonEnv = configMapE.ConfigMapData + secretDataJsonEnv = configMapE.SecretData + } + + } else if request.DeploymentWithConfig == bean.DEPLOYMENT_CONFIG_TYPE_SPECIFIC_TRIGGER { + + //fetching history and setting envLevelConfig and not appLevelConfig because history already contains merged appLevel and envLevel configs + configMapHistory, err = impl.configMapHistoryRepository.GetHistoryByPipelineIdAndWfrId(request.PipeLineId, request.WfrIdForDeploymentWithSpecificTrigger, repository3.CONFIGMAP_TYPE) + if err != nil { + impl.logger.Errorw("error in getting config map history config by pipelineId and wfrId ", "err", err, "pipelineId", request.PipeLineId, "wfrId", request.WfrIdForDeploymentWithSpecificTrigger) + return []byte("{}"), err + } + configMapJsonEnv = configMapHistory.Data + + secretHistory, err = impl.configMapHistoryRepository.GetHistoryByPipelineIdAndWfrId(request.PipeLineId, request.WfrIdForDeploymentWithSpecificTrigger, repository3.SECRET_TYPE) + if err != nil { + impl.logger.Errorw("error in getting config map history config by pipelineId and wfrId ", "err", err, "pipelineId", request.PipeLineId, "wfrId", request.WfrIdForDeploymentWithSpecificTrigger) + return []byte("{}"), err + } + secretDataJsonEnv = secretHistory.Data + } + configMapJson, err = impl.mergeUtil.ConfigMapMerge(configMapJsonApp, configMapJsonEnv) + if err != nil { + return []byte("{}"), err + } + chartMajorVersion, chartMinorVersion, err := util4.ExtractChartVersion(request.ChartVersion) + if err != nil { + impl.logger.Errorw("chart version parsing", "err", err) + return []byte("{}"), err + } + secretDataJson, err = impl.mergeUtil.ConfigSecretMerge(secretDataJsonApp, secretDataJsonEnv, chartMajorVersion, chartMinorVersion, false) + if err != nil { + return []byte("{}"), err + } + configResponseR := bean.ConfigMapRootJson{} + configResponse := bean.ConfigMapJson{} + if configMapJson != "" { + err = json.Unmarshal([]byte(configMapJson), &configResponse) + if err != nil { + return []byte("{}"), err + } + } + configResponseR.ConfigMapJson = configResponse + secretResponseR := bean.ConfigSecretRootJson{} + secretResponse := bean.ConfigSecretJson{} + if configMapJson != "" { + err = json.Unmarshal([]byte(secretDataJson), &secretResponse) + if err != nil { + return []byte("{}"), err + } + } + secretResponseR.ConfigSecretJson = secretResponse + + configMapByte, err := json.Marshal(configResponseR) + if err != nil { + return []byte("{}"), err + } + secretDataByte, err := json.Marshal(secretResponseR) + if err != nil { + return []byte("{}"), err + + } + resolvedCM, resolvedCS, snapshotCM, snapshotCS, err := impl.scopedVariableManager.ResolveCMCSTrigger(request.DeploymentWithConfig, request.Scope, configMapA.Id, configMapE.Id, configMapByte, secretDataByte, configMapHistory.Id, secretHistory.Id) + if err != nil { + return []byte("{}"), err + } + envOverride.VariableSnapshotForCM = snapshotCM + envOverride.VariableSnapshotForCS = snapshotCS + + merged, err = impl.mergeUtil.JsonPatch([]byte(resolvedCM), []byte(resolvedCS)) + + if err != nil { + return []byte("{}"), err + } + + return merged, nil +} + +func (impl *ManifestCreationServiceImpl) getReleaseOverride(envOverride *chartConfig.EnvConfigOverride, overrideRequest *bean.ValuesOverrideRequest, + artifact *repository.CiArtifact, pipelineOverride *chartConfig.PipelineOverride, strategy *chartConfig.PipelineStrategy, appMetrics *bool) (releaseOverride string, err error) { + + artifactImage := artifact.Image + imageTag := strings.Split(artifactImage, ":") + + imageTagLen := len(imageTag) + + imageName := "" + + for i := 0; i < imageTagLen-1; i++ { + if i != imageTagLen-2 { + imageName = imageName + imageTag[i] + ":" + } else { + imageName = imageName + imageTag[i] + } + } + + appId := strconv.Itoa(overrideRequest.AppId) + envId := strconv.Itoa(overrideRequest.EnvId) + + deploymentStrategy := "" + if strategy != nil { + deploymentStrategy = string(strategy.Strategy) + } + + digestConfigurationRequest := imageDigestPolicy.DigestPolicyConfigurationRequest{PipelineId: overrideRequest.PipelineId} + digestPolicyConfigurations, err := impl.imageDigestPolicyService.GetDigestPolicyConfigurations(digestConfigurationRequest) + if err != nil { + impl.logger.Errorw("error in checking if isImageDigestPolicyConfiguredForPipeline", "err", err, "pipelineId", overrideRequest.PipelineId) + return "", err + } + + if digestPolicyConfigurations.UseDigestForTrigger() { + imageTag[imageTagLen-1] = fmt.Sprintf("%s@%s", imageTag[imageTagLen-1], artifact.ImageDigest) + } + + releaseAttribute := app.ReleaseAttributes{ + Name: imageName, + Tag: imageTag[imageTagLen-1], + PipelineName: overrideRequest.PipelineName, + ReleaseVersion: strconv.Itoa(pipelineOverride.PipelineReleaseCounter), + DeploymentType: deploymentStrategy, + App: appId, + Env: envId, + AppMetrics: appMetrics, + } + override, err := util4.Tprintf(envOverride.Chart.ImageDescriptorTemplate, releaseAttribute) + if err != nil { + return "", &util.ApiError{InternalMessage: "unable to render ImageDescriptorTemplate"} + } + if overrideRequest.AdditionalOverride != nil { + userOverride, err := overrideRequest.AdditionalOverride.MarshalJSON() + if err != nil { + return "", err + } + data, err := impl.mergeUtil.JsonPatch(userOverride, []byte(override)) + if err != nil { + return "", err + } + override = string(data) + } + return override, nil +} + +func (impl *ManifestCreationServiceImpl) savePipelineOverride(overrideRequest *bean.ValuesOverrideRequest, envOverrideId int, triggeredAt time.Time) (override *chartConfig.PipelineOverride, err error) { + currentReleaseNo, err := impl.pipelineOverrideRepository.GetCurrentPipelineReleaseCounter(overrideRequest.PipelineId) + if err != nil { + return nil, err + } + po := &chartConfig.PipelineOverride{ + EnvConfigOverrideId: envOverrideId, + Status: models.CHARTSTATUS_NEW, + PipelineId: overrideRequest.PipelineId, + CiArtifactId: overrideRequest.CiArtifactId, + PipelineReleaseCounter: currentReleaseNo + 1, + CdWorkflowId: overrideRequest.CdWorkflowId, + AuditLog: sql.AuditLog{CreatedBy: overrideRequest.UserId, CreatedOn: triggeredAt, UpdatedOn: triggeredAt, UpdatedBy: overrideRequest.UserId}, + DeploymentType: overrideRequest.DeploymentType, + } + + err = impl.pipelineOverrideRepository.Save(po) + if err != nil { + return nil, err + } + err = impl.checkAndFixDuplicateReleaseNo(po) + if err != nil { + impl.logger.Errorw("error in checking release no duplicacy", "pipeline", po, "err", err) + return nil, err + } + return po, nil +} + +func (impl *ManifestCreationServiceImpl) checkAndFixDuplicateReleaseNo(override *chartConfig.PipelineOverride) error { + + uniqueVerified := false + retryCount := 0 + + for !uniqueVerified && retryCount < 5 { + retryCount = retryCount + 1 + overrides, err := impl.pipelineOverrideRepository.GetByPipelineIdAndReleaseNo(override.PipelineId, override.PipelineReleaseCounter) + if err != nil { + return err + } + if overrides[0].Id == override.Id { + uniqueVerified = true + } else { + //duplicate might be due to concurrency, lets fix it + currentReleaseNo, err := impl.pipelineOverrideRepository.GetCurrentPipelineReleaseCounter(override.PipelineId) + if err != nil { + return err + } + override.PipelineReleaseCounter = currentReleaseNo + 1 + err = impl.pipelineOverrideRepository.Save(override) + if err != nil { + return err + } + } + } + if !uniqueVerified { + return fmt.Errorf("duplicate verification retry count exide max overrideId: %d ,count: %d", override.Id, retryCount) + } + return nil +} + +func (impl *ManifestCreationServiceImpl) autoscalingCheckBeforeTrigger(ctx context.Context, appName string, namespace string, merged []byte, overrideRequest *bean.ValuesOverrideRequest) []byte { + var appId = overrideRequest.AppId + pipelineId := overrideRequest.PipelineId + var appDeploymentType = overrideRequest.DeploymentAppType + var clusterId = overrideRequest.ClusterId + deploymentType := overrideRequest.DeploymentType + templateMap := make(map[string]interface{}) + err := json.Unmarshal(merged, &templateMap) + if err != nil { + return merged + } + + hpaResourceRequest := getAutoScalingReplicaCount(templateMap, appName) + impl.logger.Debugw("autoscalingCheckBeforeTrigger", "hpaResourceRequest", hpaResourceRequest) + if hpaResourceRequest.IsEnable { + resourceManifest := make(map[string]interface{}) + if util.IsAcdApp(appDeploymentType) { + query := &application3.ApplicationResourceRequest{ + Name: &appName, + Version: &hpaResourceRequest.Version, + Group: &hpaResourceRequest.Group, + Kind: &hpaResourceRequest.Kind, + ResourceName: &hpaResourceRequest.ResourceName, + Namespace: &namespace, + } + recv, err := impl.acdClient.GetResource(ctx, query) + impl.logger.Debugw("resource manifest get replica count", "response", recv) + if err != nil { + impl.logger.Errorw("ACD Get Resource API Failed", "err", err) + middleware.AcdGetResourceCounter.WithLabelValues(strconv.Itoa(appId), namespace, appName).Inc() + return merged + } + if recv != nil && len(*recv.Manifest) > 0 { + err := json.Unmarshal([]byte(*recv.Manifest), &resourceManifest) + if err != nil { + impl.logger.Errorw("unmarshal failed for hpa check", "err", err) + return merged + } + } + } else { + version := "v2beta2" + k8sResource, err := impl.k8sCommonService.GetResource(ctx, &k8s.ResourceRequestBean{ClusterId: clusterId, + K8sRequest: &util5.K8sRequestBean{ResourceIdentifier: util5.ResourceIdentifier{Name: hpaResourceRequest.ResourceName, + Namespace: namespace, GroupVersionKind: schema.GroupVersionKind{Group: hpaResourceRequest.Group, Kind: hpaResourceRequest.Kind, Version: version}}}}) + if err != nil { + impl.logger.Errorw("error occurred while fetching resource for app", "resourceName", hpaResourceRequest.ResourceName, "err", err) + return merged + } + resourceManifest = k8sResource.ManifestResponse.Manifest.Object + } + if len(resourceManifest) > 0 { + statusMap := resourceManifest["status"].(map[string]interface{}) + currentReplicaVal := statusMap["currentReplicas"] + currentReplicaCount, err := util4.ParseFloatNumber(currentReplicaVal) + if err != nil { + impl.logger.Errorw("error occurred while parsing replica count", "currentReplicas", currentReplicaVal, "err", err) + return merged + } + + reqReplicaCount := fetchRequiredReplicaCount(currentReplicaCount, hpaResourceRequest.ReqMaxReplicas, hpaResourceRequest.ReqMinReplicas) + templateMap["replicaCount"] = reqReplicaCount + merged, err = json.Marshal(&templateMap) + if err != nil { + impl.logger.Errorw("marshaling failed for hpa check", "err", err) + return merged + } + } + } else { + impl.logger.Errorw("autoscaling is not enabled", "pipelineId", pipelineId) + } + + //check for custom chart support + if autoscalingEnabledPath, ok := templateMap[bean2.CustomAutoScalingEnabledPathKey]; ok { + if deploymentType == models.DEPLOYMENTTYPE_STOP { + merged, err = setScalingValues(templateMap, bean2.CustomAutoScalingEnabledPathKey, merged, false) + if err != nil { + impl.logger.Errorw("error occurred while setting autoscaling key", "templateMap", templateMap, "err", err) + return merged + } + merged, err = setScalingValues(templateMap, bean2.CustomAutoscalingReplicaCountPathKey, merged, 0) + if err != nil { + impl.logger.Errorw("error occurred while setting autoscaling key", "templateMap", templateMap, "err", err) + return merged + } + } else { + autoscalingEnabled := false + autoscalingEnabledValue := gjson.Get(string(merged), autoscalingEnabledPath.(string)).Value() + if val, ok := autoscalingEnabledValue.(bool); ok { + autoscalingEnabled = val + } + if autoscalingEnabled { + // extract replica count, min, max and check for required value + replicaCount, err := impl.getReplicaCountFromCustomChart(templateMap, merged) + if err != nil { + return merged + } + merged, err = setScalingValues(templateMap, bean2.CustomAutoscalingReplicaCountPathKey, merged, replicaCount) + if err != nil { + impl.logger.Errorw("error occurred while setting autoscaling key", "templateMap", templateMap, "err", err) + return merged + } + } + } + } + + return merged +} + +func (impl *ManifestCreationServiceImpl) getReplicaCountFromCustomChart(templateMap map[string]interface{}, merged []byte) (float64, error) { + autoscalingMinVal, err := extractParamValue(templateMap, bean2.CustomAutoscalingMinPathKey, merged) + if err != nil { + impl.logger.Errorw("error occurred while parsing float number", "key", bean2.CustomAutoscalingMinPathKey, "err", err) + return 0, err + } + autoscalingMaxVal, err := extractParamValue(templateMap, bean2.CustomAutoscalingMaxPathKey, merged) + if err != nil { + impl.logger.Errorw("error occurred while parsing float number", "key", bean2.CustomAutoscalingMaxPathKey, "err", err) + return 0, err + } + autoscalingReplicaCountVal, err := extractParamValue(templateMap, bean2.CustomAutoscalingReplicaCountPathKey, merged) + if err != nil { + impl.logger.Errorw("error occurred while parsing float number", "key", bean2.CustomAutoscalingReplicaCountPathKey, "err", err) + return 0, err + } + return fetchRequiredReplicaCount(autoscalingReplicaCountVal, autoscalingMaxVal, autoscalingMinVal), nil +} + +func extractParamValue(inputMap map[string]interface{}, key string, merged []byte) (float64, error) { + if _, ok := inputMap[key]; !ok { + return 0, errors.New("empty-val-err") + } + return util4.ParseFloatNumber(gjson.Get(string(merged), inputMap[key].(string)).Value()) +} + +func setScalingValues(templateMap map[string]interface{}, customScalingKey string, merged []byte, value interface{}) ([]byte, error) { + autoscalingJsonPath := templateMap[customScalingKey] + autoscalingJsonPathKey := autoscalingJsonPath.(string) + mergedRes, err := sjson.Set(string(merged), autoscalingJsonPathKey, value) + if err != nil { + return []byte{}, err + } + return []byte(mergedRes), nil +} + +func fetchRequiredReplicaCount(currentReplicaCount float64, reqMaxReplicas float64, reqMinReplicas float64) float64 { + var reqReplicaCount float64 + if currentReplicaCount <= reqMaxReplicas && currentReplicaCount >= reqMinReplicas { + reqReplicaCount = currentReplicaCount + } else if currentReplicaCount > reqMaxReplicas { + reqReplicaCount = reqMaxReplicas + } else if currentReplicaCount < reqMinReplicas { + reqReplicaCount = reqMinReplicas + } + return reqReplicaCount +} + +func getAutoScalingReplicaCount(templateMap map[string]interface{}, appName string) *util4.HpaResourceRequest { + hasOverride := false + if _, ok := templateMap[bean3.FullnameOverride]; ok { + appNameOverride := templateMap[bean3.FullnameOverride].(string) + if len(appNameOverride) > 0 { + appName = appNameOverride + hasOverride = true + } + } + if !hasOverride { + if _, ok := templateMap[bean3.NameOverride]; ok { + nameOverride := templateMap[bean3.NameOverride].(string) + if len(nameOverride) > 0 { + appName = fmt.Sprintf("%s-%s", appName, nameOverride) + } + } + } + hpaResourceRequest := &util4.HpaResourceRequest{} + hpaResourceRequest.Version = "" + hpaResourceRequest.Group = autoscaling.ServiceName + hpaResourceRequest.Kind = bean3.HorizontalPodAutoscaler + if _, ok := templateMap[bean3.KedaAutoscaling]; ok { + as := templateMap[bean3.KedaAutoscaling] + asd := as.(map[string]interface{}) + if _, ok := asd[bean3.Enabled]; ok { + enable := asd[bean3.Enabled].(bool) + if enable { + hpaResourceRequest.IsEnable = enable + hpaResourceRequest.ReqReplicaCount = templateMap[bean3.ReplicaCount].(float64) + hpaResourceRequest.ReqMaxReplicas = asd["maxReplicaCount"].(float64) + hpaResourceRequest.ReqMinReplicas = asd["minReplicaCount"].(float64) + hpaResourceRequest.ResourceName = fmt.Sprintf("%s-%s-%s", "keda-hpa", appName, "keda") + return hpaResourceRequest + } + } + } + + if _, ok := templateMap[autoscaling.ServiceName]; ok { + as := templateMap[autoscaling.ServiceName] + asd := as.(map[string]interface{}) + if _, ok := asd[bean3.Enabled]; ok { + enable := asd[bean3.Enabled].(bool) + if enable { + hpaResourceRequest.IsEnable = asd[bean3.Enabled].(bool) + hpaResourceRequest.ReqReplicaCount = templateMap[bean3.ReplicaCount].(float64) + hpaResourceRequest.ReqMaxReplicas = asd["MaxReplicas"].(float64) + hpaResourceRequest.ReqMinReplicas = asd["MinReplicas"].(float64) + hpaResourceRequest.ResourceName = fmt.Sprintf("%s-%s", appName, "hpa") + return hpaResourceRequest + } + } + } + return hpaResourceRequest + +} + +func createConfigMapAndSecretJsonRequest(overrideRequest *bean.ValuesOverrideRequest, envOverride *chartConfig.EnvConfigOverride, chartVersion string, scope resourceQualifiers.Scope) bean3.ConfigMapAndSecretJsonV2 { + request := bean3.ConfigMapAndSecretJsonV2{ + AppId: overrideRequest.AppId, + EnvId: envOverride.TargetEnvironment, + PipeLineId: overrideRequest.PipelineId, + ChartVersion: chartVersion, + DeploymentWithConfig: overrideRequest.DeploymentWithConfig, + WfrIdForDeploymentWithSpecificTrigger: overrideRequest.WfrIdForDeploymentWithSpecificTrigger, + Scope: scope, + } + return request +} + +func getScopeForVariables(overrideRequest *bean.ValuesOverrideRequest, envOverride *chartConfig.EnvConfigOverride) resourceQualifiers.Scope { + scope := resourceQualifiers.Scope{ + AppId: overrideRequest.AppId, + EnvId: envOverride.TargetEnvironment, + ClusterId: envOverride.Environment.Id, + SystemMetadata: &resourceQualifiers.SystemMetadata{ + EnvironmentName: envOverride.Environment.Name, + ClusterName: envOverride.Environment.Cluster.ClusterName, + Namespace: envOverride.Environment.Namespace, + ImageTag: util3.GetImageTagFromImage(overrideRequest.Image), + AppName: overrideRequest.AppName, + Image: overrideRequest.Image, + }, + } + return scope +} diff --git a/pkg/deployment/manifest/bean/bean.go b/pkg/deployment/manifest/bean/bean.go new file mode 100644 index 0000000000..9ef1930905 --- /dev/null +++ b/pkg/deployment/manifest/bean/bean.go @@ -0,0 +1,25 @@ +package bean + +import ( + "github.com/devtron-labs/devtron/api/bean" + "github.com/devtron-labs/devtron/pkg/resourceQualifiers" +) + +const ( + FullnameOverride = "fullnameOverride" + NameOverride = "nameOverride" + KedaAutoscaling = "kedaAutoscaling" + HorizontalPodAutoscaler = "HorizontalPodAutoscaler" + Enabled = "enabled" + ReplicaCount = "replicaCount" +) + +type ConfigMapAndSecretJsonV2 struct { + AppId int + EnvId int + PipeLineId int + ChartVersion string + DeploymentWithConfig bean.DeploymentConfigurationType + WfrIdForDeploymentWithSpecificTrigger int + Scope resourceQualifiers.Scope +} diff --git a/pkg/deployment/manifest/deployedAppMetrics/wire_appMetrics.go b/pkg/deployment/manifest/deployedAppMetrics/wire_appMetrics.go new file mode 100644 index 0000000000..025eec7466 --- /dev/null +++ b/pkg/deployment/manifest/deployedAppMetrics/wire_appMetrics.go @@ -0,0 +1,16 @@ +package deployedAppMetrics + +import ( + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/repository" + "github.com/google/wire" +) + +var AppMetricsWireSet = wire.NewSet( + repository.NewAppLevelMetricsRepositoryImpl, + wire.Bind(new(repository.AppLevelMetricsRepository), new(*repository.AppLevelMetricsRepositoryImpl)), + repository.NewEnvLevelAppMetricsRepositoryImpl, + wire.Bind(new(repository.EnvLevelAppMetricsRepository), new(*repository.EnvLevelAppMetricsRepositoryImpl)), + + NewDeployedAppMetricsServiceImpl, + wire.Bind(new(DeployedAppMetricsService), new(*DeployedAppMetricsServiceImpl)), +) diff --git a/pkg/deployment/manifest/deploymentTemplate/DeploymentTemplateService.go b/pkg/deployment/manifest/deploymentTemplate/DeploymentTemplateService.go index 2c43ce535a..34b71a70e5 100644 --- a/pkg/deployment/manifest/deploymentTemplate/DeploymentTemplateService.go +++ b/pkg/deployment/manifest/deploymentTemplate/DeploymentTemplateService.go @@ -1,16 +1,126 @@ package deploymentTemplate -import "go.uber.org/zap" +import ( + "context" + "fmt" + "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" + "github.com/devtron-labs/devtron/internal/util" + chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" + bean4 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/bean" + "go.opentelemetry.io/otel" + "go.uber.org/zap" + chart2 "k8s.io/helm/pkg/proto/hapi/chart" + "os" + "path" + "path/filepath" + "strings" +) type DeploymentTemplateService interface { + BuildChartAndGetPath(appName string, envOverride *chartConfig.EnvConfigOverride, ctx context.Context) (string, error) } type DeploymentTemplateServiceImpl struct { - logger *zap.SugaredLogger + logger *zap.SugaredLogger + chartRefService chartRef.ChartRefService + chartTemplateService util.ChartTemplateService + + chartRepository chartRepoRepository.ChartRepository } -func NewDeploymentTemplateServiceImpl(logger *zap.SugaredLogger) *DeploymentTemplateServiceImpl { +func NewDeploymentTemplateServiceImpl(logger *zap.SugaredLogger, + chartRefService chartRef.ChartRefService, + chartTemplateService util.ChartTemplateService, + chartRepository chartRepoRepository.ChartRepository) *DeploymentTemplateServiceImpl { return &DeploymentTemplateServiceImpl{ - logger: logger, + logger: logger, + chartRefService: chartRefService, + chartTemplateService: chartTemplateService, + chartRepository: chartRepository, + } +} + +func (impl *DeploymentTemplateServiceImpl) BuildChartAndGetPath(appName string, envOverride *chartConfig.EnvConfigOverride, ctx context.Context) (string, error) { + if !strings.HasSuffix(envOverride.Chart.ChartLocation, fmt.Sprintf("%s%s", "/", envOverride.Chart.ChartVersion)) { + _, span := otel.Tracer("orchestrator").Start(ctx, "autoHealChartLocationInChart") + err := impl.autoHealChartLocationInChart(ctx, envOverride) + span.End() + if err != nil { + return "", err + } + } + chartMetaData := &chart2.Metadata{ + Name: appName, + Version: envOverride.Chart.ChartVersion, } + referenceTemplatePath := path.Join(bean4.RefChartDirPath, envOverride.Chart.ReferenceTemplate) + // Load custom charts to referenceTemplatePath if not exists + if _, err := os.Stat(referenceTemplatePath); os.IsNotExist(err) { + chartRefValue, err := impl.chartRefService.FindById(envOverride.Chart.ChartRefId) + if err != nil { + impl.logger.Errorw("error in fetching ChartRef data", "err", err) + return "", err + } + if chartRefValue.ChartData != nil { + chartInfo, err := impl.chartRefService.ExtractChartIfMissing(chartRefValue.ChartData, bean4.RefChartDirPath, chartRefValue.Location) + if chartInfo != nil && chartInfo.TemporaryFolder != "" { + err1 := os.RemoveAll(chartInfo.TemporaryFolder) + if err1 != nil { + impl.logger.Errorw("error in deleting temp dir ", "err", err) + } + } + return "", err + } + } + _, span := otel.Tracer("orchestrator").Start(ctx, "chartTemplateService.BuildChart") + tempReferenceTemplateDir, err := impl.chartTemplateService.BuildChart(ctx, chartMetaData, referenceTemplatePath) + span.End() + if err != nil { + return "", err + } + return tempReferenceTemplateDir, nil +} + +func (impl *DeploymentTemplateServiceImpl) autoHealChartLocationInChart(ctx context.Context, envOverride *chartConfig.EnvConfigOverride) error { + chartId := envOverride.Chart.Id + impl.logger.Infow("auto-healing: Chart location in chart not correct. modifying ", "chartId", chartId, + "current chartLocation", envOverride.Chart.ChartLocation, "current chartVersion", envOverride.Chart.ChartVersion) + + // get chart from DB (getting it from DB because envOverride.Chart does not have full row of DB) + _, span := otel.Tracer("orchestrator").Start(ctx, "chartRepository.FindById") + chart, err := impl.chartRepository.FindById(chartId) + span.End() + if err != nil { + impl.logger.Errorw("error occurred while fetching chart from DB", "chartId", chartId, "err", err) + return err + } + + // get chart ref from DB (to get location) + chartRefId := chart.ChartRefId + _, span = otel.Tracer("orchestrator").Start(ctx, "chartRefRepository.FindById") + chartRefDto, err := impl.chartRefService.FindById(chartRefId) + span.End() + if err != nil { + impl.logger.Errorw("error occurred while fetching chartRef from DB", "chartRefId", chartRefId, "err", err) + return err + } + + // build new chart location + newChartLocation := filepath.Join(chartRefDto.Location, envOverride.Chart.ChartVersion) + impl.logger.Infow("new chart location build", "chartId", chartId, "newChartLocation", newChartLocation) + + // update chart in DB + chart.ChartLocation = newChartLocation + _, span = otel.Tracer("orchestrator").Start(ctx, "chartRepository.Update") + err = impl.chartRepository.Update(chart) + span.End() + if err != nil { + impl.logger.Errorw("error occurred while saving chart into DB", "chartId", chartId, "err", err) + return err + } + + // update newChartLocation in model + envOverride.Chart.ChartLocation = newChartLocation + return nil } diff --git a/pkg/deployment/manifest/deploymentTemplate/wire_deploymentTemplate.go b/pkg/deployment/manifest/deploymentTemplate/wire_deploymentTemplate.go new file mode 100644 index 0000000000..dd8e42f202 --- /dev/null +++ b/pkg/deployment/manifest/deploymentTemplate/wire_deploymentTemplate.go @@ -0,0 +1,15 @@ +package deploymentTemplate + +import ( + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" + "github.com/google/wire" +) + +var DeploymentTemplateWireSet = wire.NewSet( + NewDeploymentTemplateServiceImpl, + wire.Bind(new(DeploymentTemplateService), new(*DeploymentTemplateServiceImpl)), + NewDeploymentTemplateValidationServiceImpl, + wire.Bind(new(DeploymentTemplateValidationService), new(*DeploymentTemplateValidationServiceImpl)), + chartRef.NewChartRefServiceImpl, + wire.Bind(new(chartRef.ChartRefService), new(*chartRef.ChartRefServiceImpl)), +) diff --git a/pkg/deployment/manifest/wire_deployment_manifest.go b/pkg/deployment/manifest/wire_deployment_manifest.go index 425fbbd19c..0b2ca41393 100644 --- a/pkg/deployment/manifest/wire_deployment_manifest.go +++ b/pkg/deployment/manifest/wire_deployment_manifest.go @@ -2,24 +2,14 @@ package manifest import ( "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" - "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/repository" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate" - "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" "github.com/google/wire" ) var DeploymentManifestWireSet = wire.NewSet( - repository.NewAppLevelMetricsRepositoryImpl, - wire.Bind(new(repository.AppLevelMetricsRepository), new(*repository.AppLevelMetricsRepositoryImpl)), - repository.NewEnvLevelAppMetricsRepositoryImpl, - wire.Bind(new(repository.EnvLevelAppMetricsRepository), new(*repository.EnvLevelAppMetricsRepositoryImpl)), + deployedAppMetrics.AppMetricsWireSet, + deploymentTemplate.DeploymentTemplateWireSet, - deployedAppMetrics.NewDeployedAppMetricsServiceImpl, - wire.Bind(new(deployedAppMetrics.DeployedAppMetricsService), new(*deployedAppMetrics.DeployedAppMetricsServiceImpl)), - deploymentTemplate.NewDeploymentTemplateServiceImpl, - wire.Bind(new(deploymentTemplate.DeploymentTemplateService), new(*deploymentTemplate.DeploymentTemplateServiceImpl)), - deploymentTemplate.NewDeploymentTemplateValidationServiceImpl, - wire.Bind(new(deploymentTemplate.DeploymentTemplateValidationService), new(*deploymentTemplate.DeploymentTemplateValidationServiceImpl)), - chartRef.NewChartRefServiceImpl, - wire.Bind(new(chartRef.ChartRefService), new(*chartRef.ChartRefServiceImpl)), + NewManifestCreationServiceImpl, + wire.Bind(new(ManifestCreationService), new(*ManifestCreationServiceImpl)), ) diff --git a/pkg/deployment/trigger/devtronApps/TriggerService.go b/pkg/deployment/trigger/devtronApps/TriggerService.go index f33ac80415..02f1aff570 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService.go +++ b/pkg/deployment/trigger/devtronApps/TriggerService.go @@ -2,11 +2,8 @@ package devtronApps import ( "context" - "encoding/json" "errors" "fmt" - application3 "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" - "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" pubsub "github.com/devtron-labs/common-lib/pubsub-lib" util5 "github.com/devtron-labs/common-lib/utils/k8s" bean3 "github.com/devtron-labs/devtron/api/bean" @@ -14,7 +11,7 @@ import ( "github.com/devtron-labs/devtron/api/helm-app/gRPC" client2 "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/client/argocdServer" - application2 "github.com/devtron-labs/devtron/client/argocdServer/application" + bean7 "github.com/devtron-labs/devtron/client/argocdServer/bean" client "github.com/devtron-labs/devtron/client/events" "github.com/devtron-labs/devtron/internal/middleware" repository3 "github.com/devtron-labs/devtron/internal/sql/repository" @@ -34,6 +31,7 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline/repository" "github.com/devtron-labs/devtron/pkg/sql" "github.com/devtron-labs/devtron/pkg/workflow/cd" + util3 "github.com/devtron-labs/devtron/util" util2 "github.com/devtron-labs/devtron/util/event" "github.com/go-pg/pg" "go.opentelemetry.io/otel" @@ -49,6 +47,9 @@ import ( type TriggerService interface { TriggerRelease(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, ctx context.Context, triggeredAt time.Time, triggeredBy int32) (releaseNo int, manifest []byte, err error) + + //TODO: make this method private and move all usages in this service since TriggerService should own if async mode is enabled and if yes then how to act on it + IsDevtronAsyncInstallModeEnabled(deploymentAppType string) bool } type TriggerServiceImpl struct { @@ -57,13 +58,12 @@ type TriggerServiceImpl struct { gitOpsManifestPushService app.GitOpsPushService argoK8sClient argocdServer.ArgoK8sClient ACDConfig *argocdServer.ACDConfig - acdClient application2.ServiceClient argoClientWrapperService argocdServer.ArgoClientWrapperService pipelineStatusTimelineService status.PipelineStatusTimelineService chartTemplateService util.ChartTemplateService - appService app.AppService eventFactory client.EventFactory eventClient client.EventClient + globalEnvVariables *util3.GlobalEnvVariables helmAppService client2.HelmAppService @@ -83,15 +83,14 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, cdWorkflowCommonService cd gitOpsManifestPushService app.GitOpsPushService, argoK8sClient argocdServer.ArgoK8sClient, ACDConfig *argocdServer.ACDConfig, - acdClient application2.ServiceClient, argoClientWrapperService argocdServer.ArgoClientWrapperService, pipelineStatusTimelineService status.PipelineStatusTimelineService, chartTemplateService util.ChartTemplateService, - appService app.AppService, helmAppService client2.HelmAppService, helmAppClient gRPC.HelmAppClient, eventFactory client.EventFactory, eventClient client.EventClient, + globalEnvVariables *util3.GlobalEnvVariables, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, imageScanHistoryRepository security.ImageScanHistoryRepository, imageScanDeployInfoRepository security.ImageScanDeployInfoRepository, @@ -106,17 +105,14 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, cdWorkflowCommonService cd gitOpsManifestPushService: gitOpsManifestPushService, argoK8sClient: argoK8sClient, ACDConfig: ACDConfig, - acdClient: acdClient, argoClientWrapperService: argoClientWrapperService, pipelineStatusTimelineService: pipelineStatusTimelineService, chartTemplateService: chartTemplateService, - appService: appService, helmAppService: helmAppService, - eventClient: eventClient, eventFactory: eventFactory, - - helmAppClient: helmAppClient, - + eventClient: eventClient, + globalEnvVariables: globalEnvVariables, + helmAppClient: helmAppClient, ciPipelineMaterialRepository: ciPipelineMaterialRepository, imageScanHistoryRepository: imageScanHistoryRepository, imageScanDeployInfoRepository: imageScanDeployInfoRepository, @@ -333,7 +329,7 @@ func (impl *TriggerServiceImpl) createHelmAppForCdPipeline(overrideRequest *bean HistoryMax: impl.helmAppService.GetRevisionHistoryMaxValue(bean6.SOURCE_DEVTRON_APP), ChartContent: &gRPC.ChartContent{Content: referenceChartByte}, } - if impl.appService.IsDevtronAsyncInstallModeEnabled(bean.Helm) { + if impl.IsDevtronAsyncInstallModeEnabled(bean.Helm) { req.RunInCtx = true } // For cases where helm release was not found, kubelink will install the same configuration @@ -452,7 +448,7 @@ func (impl *TriggerServiceImpl) updateArgoPipeline(pipeline *pipelineConfig.Pipe } argoAppName := pipeline.DeploymentAppName impl.logger.Infow("received payload, updateArgoPipeline", "appId", pipeline.AppId, "pipelineName", pipeline.Name, "envId", envOverride.TargetEnvironment, "argoAppName", argoAppName, "context", ctx) - argoApplication, err := impl.acdClient.Get(ctx, &application3.ApplicationQuery{Name: &argoAppName}) + argoApplication, err := impl.argoClientWrapperService.GetArgoAppByName(ctx, argoAppName) if err != nil { impl.logger.Errorw("no argo app exists", "app", argoAppName, "pipeline", pipeline.Name) return false, err @@ -462,19 +458,19 @@ func (impl *TriggerServiceImpl) updateArgoPipeline(pipeline *pipelineConfig.Pipe if appStatus.Code() == codes.OK { impl.logger.Debugw("argo app exists", "app", argoAppName, "pipeline", pipeline.Name) if argoApplication.Spec.Source.Path != envOverride.Chart.ChartLocation || argoApplication.Spec.Source.TargetRevision != "master" { - patchReq := v1alpha1.Application{Spec: v1alpha1.ApplicationSpec{Source: &v1alpha1.ApplicationSource{Path: envOverride.Chart.ChartLocation, RepoURL: envOverride.Chart.GitRepoUrl, TargetRevision: "master"}}} - reqbyte, err := json.Marshal(patchReq) - if err != nil { - impl.logger.Errorw("error in creating patch", "err", err) + patchRequestDto := &bean7.ArgoCdAppPatchReqDto{ + ArgoAppName: argoAppName, + ChartLocation: envOverride.Chart.ChartLocation, + GitRepoUrl: envOverride.Chart.GitRepoUrl, + TargetRevision: bean7.TargetRevisionMaster, + PatchType: bean7.PatchTypeMerge, } - reqString := string(reqbyte) - patchType := "merge" - _, err = impl.acdClient.Patch(ctx, &application3.ApplicationPatchRequest{Patch: &reqString, Name: &argoAppName, PatchType: &patchType}) + err = impl.argoClientWrapperService.PatchArgoCdApp(ctx, patchRequestDto) if err != nil { - impl.logger.Errorw("error in creating argo pipeline ", "name", pipeline.Name, "patch", string(reqbyte), "err", err) + impl.logger.Errorw("error in patching argo pipeline", "err", err, "req", patchRequestDto) return false, err } - impl.logger.Debugw("pipeline update req ", "res", patchReq) + impl.logger.Debugw("pipeline update req", "res", patchRequestDto) } else { impl.logger.Debug("pipeline no need to update ") } @@ -521,7 +517,7 @@ func (impl *TriggerServiceImpl) createArgoApplicationIfRequired(appId int, envCo TargetNamespace: appNamespace, TargetServer: envModel.Cluster.ServerUrl, Project: "default", - ValuesFile: impl.getValuesFileForEnv(envModel.Id), + ValuesFile: getValuesFileForEnv(envModel.Id), RepoPath: chart.ChartLocation, RepoUrl: chart.GitRepoUrl, AutoSyncEnabled: impl.ACDConfig.ArgoCDAutoSyncEnabled, @@ -540,7 +536,7 @@ func (impl *TriggerServiceImpl) createArgoApplicationIfRequired(appId int, envCo } } -func (impl *TriggerServiceImpl) getValuesFileForEnv(environmentId int) string { +func getValuesFileForEnv(environmentId int) string { return fmt.Sprintf("_%d-values.yaml", environmentId) //-{envId}-values.yaml } @@ -561,7 +557,7 @@ func (impl *TriggerServiceImpl) helmInstallReleaseWithCustomChart(ctx context.Co ChartContent: &gRPC.ChartContent{Content: referenceChartByte}, ReleaseIdentifier: releaseIdentifier, } - if impl.appService.IsDevtronAsyncInstallModeEnabled(bean.Helm) { + if impl.IsDevtronAsyncInstallModeEnabled(bean.Helm) { helmInstallRequest.RunInCtx = true } // Request exec @@ -664,3 +660,8 @@ func (impl *TriggerServiceImpl) markImageScanDeployed(appId int, envId int, imag } return err } + +func (impl *TriggerServiceImpl) IsDevtronAsyncInstallModeEnabled(deploymentAppType string) bool { + return impl.globalEnvVariables.EnableAsyncInstallDevtronChart && + deploymentAppType == bean.Helm +} diff --git a/pkg/eventProcessor/out/WorkflowEventPublishService.go b/pkg/eventProcessor/out/WorkflowEventPublishService.go index 7ef9f1a6ee..834ac23554 100644 --- a/pkg/eventProcessor/out/WorkflowEventPublishService.go +++ b/pkg/eventProcessor/out/WorkflowEventPublishService.go @@ -1,36 +1,152 @@ package out import ( + "context" "encoding/json" pubsub "github.com/devtron-labs/common-lib/pubsub-lib" + bean3 "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/pkg/app/status" + "github.com/devtron-labs/devtron/pkg/deployment/manifest" + bean7 "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" "github.com/devtron-labs/devtron/pkg/eventProcessor/out/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/history" + "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/sql" + "github.com/devtron-labs/devtron/pkg/workflow/cd" + util4 "github.com/devtron-labs/devtron/util" + "go.opentelemetry.io/otel" "go.uber.org/zap" "time" ) type WorkflowEventPublishService interface { + TriggerHelmAsyncRelease(overrideRequest *bean3.ValuesOverrideRequest, ctx context.Context, triggeredAt time.Time, + triggeredBy int32) (releaseNo int, manifest []byte, err error) TriggerBulkDeploymentAsync(requests []*bean.BulkTriggerRequest, UserId int32) (interface{}, error) } type WorkflowEventPublishServiceImpl struct { - logger *zap.SugaredLogger - pubSubClient *pubsub.PubSubClientServiceImpl + logger *zap.SugaredLogger + pubSubClient *pubsub.PubSubClientServiceImpl + cdWorkflowCommonService cd.CdWorkflowCommonService + deployedConfigurationHistoryService history.DeployedConfigurationHistoryService + manifestCreationService manifest.ManifestCreationService + pipelineStatusTimelineService status.PipelineStatusTimelineService + config *types.CdConfig + cdWorkflowRepository pipelineConfig.CdWorkflowRepository + pipelineRepository pipelineConfig.PipelineRepository } func NewWorkflowEventPublishServiceImpl(logger *zap.SugaredLogger, pubSubClient *pubsub.PubSubClientServiceImpl, - cdWorkflowRepository pipelineConfig.CdWorkflowRepository) (*WorkflowEventPublishServiceImpl, error) { + cdWorkflowCommonService cd.CdWorkflowCommonService, + deployedConfigurationHistoryService history.DeployedConfigurationHistoryService, + manifestCreationService manifest.ManifestCreationService, + pipelineStatusTimelineService status.PipelineStatusTimelineService, + + cdWorkflowRepository pipelineConfig.CdWorkflowRepository, + pipelineRepository pipelineConfig.PipelineRepository) (*WorkflowEventPublishServiceImpl, error) { + config, err := types.GetCdConfig() + if err != nil { + return nil, err + } impl := &WorkflowEventPublishServiceImpl{ - logger: logger, - pubSubClient: pubSubClient, + logger: logger, + pubSubClient: pubSubClient, + cdWorkflowCommonService: cdWorkflowCommonService, + deployedConfigurationHistoryService: deployedConfigurationHistoryService, + manifestCreationService: manifestCreationService, + pipelineStatusTimelineService: pipelineStatusTimelineService, + config: config, + cdWorkflowRepository: cdWorkflowRepository, + pipelineRepository: pipelineRepository, } return impl, nil } +// TriggerHelmAsyncRelease will publish async helm Install/Upgrade request event for Devtron App releases +func (impl *WorkflowEventPublishServiceImpl) TriggerHelmAsyncRelease(overrideRequest *bean3.ValuesOverrideRequest, ctx context.Context, triggeredAt time.Time, triggeredBy int32) (releaseNo int, manifest []byte, err error) { + // build merged values and save PCO history for the release + valuesOverrideResponse, err := impl.manifestCreationService.GetValuesOverrideForTrigger(overrideRequest, triggeredAt, ctx) + _, span := otel.Tracer("orchestrator").Start(ctx, "CreateHistoriesForDeploymentTrigger") + // save triggered deployment history + err1 := impl.deployedConfigurationHistoryService.CreateHistoriesForDeploymentTrigger(valuesOverrideResponse.Pipeline, valuesOverrideResponse.PipelineStrategy, valuesOverrideResponse.EnvOverride, triggeredAt, triggeredBy) + if err1 != nil { + impl.logger.Errorw("error in saving histories for trigger", "err", err1, "pipelineId", valuesOverrideResponse.Pipeline.Id, "wfrId", overrideRequest.WfrId) + } + span.End() + if err != nil { + impl.logger.Errorw("error in fetching values for trigger", "err", err) + return releaseNo, manifest, err + } + + event := &bean7.AsyncCdDeployEvent{ + ValuesOverrideRequest: overrideRequest, + TriggeredAt: triggeredAt, + TriggeredBy: triggeredBy, + } + payload, err := json.Marshal(event) + if err != nil { + impl.logger.Errorw("failed to marshal helm async CD deploy event request", "request", event, "err", err) + return 0, manifest, err + } + + // publish nats event for async installation + err = impl.pubSubClient.Publish(pubsub.DEVTRON_CHART_INSTALL_TOPIC, string(payload)) + if err != nil { + impl.logger.Errorw("failed to publish trigger request event", "topic", pubsub.DEVTRON_CHART_INSTALL_TOPIC, "payload", payload, "err", err) + //update workflow runner status, used in app workflow view + err1 = impl.cdWorkflowCommonService.UpdateCDWorkflowRunnerStatus(ctx, overrideRequest, triggeredAt, pipelineConfig.WorkflowFailed, err.Error()) + if err1 != nil { + impl.logger.Errorw("error in updating the workflow runner status, TriggerHelmAsyncRelease", "err", err1) + } + return 0, manifest, err + } + + //update workflow runner status, used in app workflow view + err = impl.cdWorkflowCommonService.UpdateCDWorkflowRunnerStatus(ctx, overrideRequest, triggeredAt, pipelineConfig.WorkflowInQueue, "") + if err != nil { + impl.logger.Errorw("error in updating the workflow runner status, TriggerHelmAsyncRelease", "err", err) + return 0, manifest, err + } + err = impl.UpdatePreviousQueuedRunnerStatus(overrideRequest.WfrId, overrideRequest.PipelineId, triggeredBy) + if err != nil { + impl.logger.Errorw("error in updating the previous queued workflow runner status, TriggerHelmAsyncRelease", "err", err) + return 0, manifest, err + } + return 0, manifest, nil +} + +func (impl *WorkflowEventPublishServiceImpl) UpdatePreviousQueuedRunnerStatus(cdWfrId, pipelineId int, triggeredBy int32) error { + cdWfrs, err := impl.cdWorkflowRepository.UpdatePreviousQueuedRunnerStatus(cdWfrId, pipelineId, triggeredBy) + if err != nil { + impl.logger.Errorw("error on update previous queued cd workflow runner, UpdatePreviousQueuedRunnerStatus", "cdWfrId", cdWfrId, "err", err) + return err + } + for _, cdWfr := range cdWfrs { + err = impl.pipelineStatusTimelineService.MarkPipelineStatusTimelineFailed(cdWfr.Id, pipelineConfig.NEW_DEPLOYMENT_INITIATED) + if err != nil { + impl.logger.Errorw("error updating CdPipelineStatusTimeline, UpdatePreviousQueuedRunnerStatus", "err", err) + return err + } + if cdWfr.CdWorkflow == nil { + pipeline, err := impl.pipelineRepository.FindById(pipelineId) + if err != nil { + impl.logger.Errorw("error in fetching cd pipeline, UpdatePreviousQueuedRunnerStatus", "pipelineId", pipelineId, "err", err) + return err + } + cdWfr.CdWorkflow = &pipelineConfig.CdWorkflow{ + Pipeline: pipeline, + } + } + util4.TriggerCDMetrics(pipelineConfig.GetTriggerMetricsFromRunnerObj(cdWfr), impl.config.ExposeCDMetrics) + } + return nil +} + func (impl *WorkflowEventPublishServiceImpl) TriggerBulkDeploymentAsync(requests []*bean.BulkTriggerRequest, UserId int32) (interface{}, error) { var cdWorkflows []*pipelineConfig.CdWorkflow for _, request := range requests { diff --git a/pkg/pipeline/BuildPipelineConfigService.go b/pkg/pipeline/BuildPipelineConfigService.go index 511d302517..3f3369620b 100644 --- a/pkg/pipeline/BuildPipelineConfigService.go +++ b/pkg/pipeline/BuildPipelineConfigService.go @@ -108,31 +108,29 @@ type CiPipelineConfigService interface { } type CiPipelineConfigServiceImpl struct { - logger *zap.SugaredLogger - ciTemplateService CiTemplateService - materialRepo pipelineConfig.MaterialRepository - ciPipelineRepository pipelineConfig.CiPipelineRepository - ciConfig *types.CiCdConfig - attributesService attributes.AttributesService - ciWorkflowRepository pipelineConfig.CiWorkflowRepository - appWorkflowRepository appWorkflow.AppWorkflowRepository - pipelineStageService PipelineStageService - pipelineRepository pipelineConfig.PipelineRepository - appRepo app2.AppRepository - dockerArtifactStoreRepository dockerRegistryRepository.DockerArtifactStoreRepository - ciCdPipelineOrchestrator CiCdPipelineOrchestrator - ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository - CiTemplateHistoryService history.CiTemplateHistoryService - securityConfig *SecurityConfig - ecrConfig *EcrConfig - ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository - resourceGroupService resourceGroup2.ResourceGroupService - enforcerUtil rbac.EnforcerUtil - customTagService CustomTagService - deployedConfigurationHistoryService history.DeployedConfigurationHistoryService - ciPipelineHistoryService history.CiPipelineHistoryService - cdWorkflowRepository pipelineConfig.CdWorkflowRepository - buildPipelineSwitchService BuildPipelineSwitchService + logger *zap.SugaredLogger + ciTemplateService CiTemplateService + materialRepo pipelineConfig.MaterialRepository + ciPipelineRepository pipelineConfig.CiPipelineRepository + ciConfig *types.CiCdConfig + attributesService attributes.AttributesService + ciWorkflowRepository pipelineConfig.CiWorkflowRepository + appWorkflowRepository appWorkflow.AppWorkflowRepository + pipelineStageService PipelineStageService + pipelineRepository pipelineConfig.PipelineRepository + appRepo app2.AppRepository + dockerArtifactStoreRepository dockerRegistryRepository.DockerArtifactStoreRepository + ciCdPipelineOrchestrator CiCdPipelineOrchestrator + ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository + CiTemplateHistoryService history.CiTemplateHistoryService + securityConfig *SecurityConfig + ecrConfig *EcrConfig + ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository + resourceGroupService resourceGroup2.ResourceGroupService + enforcerUtil rbac.EnforcerUtil + customTagService CustomTagService + cdWorkflowRepository pipelineConfig.CdWorkflowRepository + buildPipelineSwitchService BuildPipelineSwitchService } func NewCiPipelineConfigServiceImpl(logger *zap.SugaredLogger, @@ -155,7 +153,6 @@ func NewCiPipelineConfigServiceImpl(logger *zap.SugaredLogger, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, resourceGroupService resourceGroup2.ResourceGroupService, customTagService CustomTagService, - ciPipelineHistoryService history.CiPipelineHistoryService, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, buildPipelineSwitchService BuildPipelineSwitchService, ) *CiPipelineConfigServiceImpl { @@ -187,7 +184,6 @@ func NewCiPipelineConfigServiceImpl(logger *zap.SugaredLogger, resourceGroupService: resourceGroupService, securityConfig: securityConfig, customTagService: customTagService, - ciPipelineHistoryService: ciPipelineHistoryService, cdWorkflowRepository: cdWorkflowRepository, buildPipelineSwitchService: buildPipelineSwitchService, } diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index 4755bca229..79443531c8 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -23,30 +23,25 @@ import ( "fmt" bean6 "github.com/devtron-labs/devtron/api/helm-app/bean" client2 "github.com/devtron-labs/devtron/api/helm-app/service" - "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" - "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" + "github.com/devtron-labs/devtron/pkg/deployment/manifest" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps" bean7 "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" + "github.com/devtron-labs/devtron/pkg/eventProcessor/out" "github.com/devtron-labs/devtron/pkg/workflow/cd" "strconv" "strings" "sync" "time" - application3 "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" - "github.com/aws/aws-sdk-go/service/autoscaling" blob_storage "github.com/devtron-labs/common-lib/blob-storage" "github.com/devtron-labs/common-lib/pubsub-lib/model" util5 "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/common-lib/utils/k8s/health" "github.com/devtron-labs/devtron/client/argocdServer/application" - application2 "github.com/devtron-labs/devtron/client/argocdServer/application" gitSensorClient "github.com/devtron-labs/devtron/client/gitSensor" - "github.com/devtron-labs/devtron/internal/middleware" "github.com/devtron-labs/devtron/pkg/app/status" "github.com/devtron-labs/devtron/pkg/auth/user" - "github.com/devtron-labs/devtron/pkg/chartRepo/repository" - "github.com/devtron-labs/devtron/pkg/dockerRegistry" "github.com/devtron-labs/devtron/pkg/imageDigestPolicy" "github.com/devtron-labs/devtron/pkg/k8s" bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" @@ -56,16 +51,11 @@ import ( "github.com/devtron-labs/devtron/pkg/resourceQualifiers" serverBean "github.com/devtron-labs/devtron/pkg/server/bean" "github.com/devtron-labs/devtron/pkg/variables" - "github.com/devtron-labs/devtron/pkg/variables/parsers" repository5 "github.com/devtron-labs/devtron/pkg/variables/repository" util4 "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/argo" - errors2 "github.com/juju/errors" "github.com/pkg/errors" - "github.com/tidwall/gjson" - "github.com/tidwall/sjson" "go.opentelemetry.io/otel" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/utils/strings/slices" "github.com/devtron-labs/devtron/internal/sql/repository/appWorkflow" @@ -120,7 +110,6 @@ type WorkflowDagExecutorImpl struct { pipelineRepository pipelineConfig.PipelineRepository cdWorkflowRepository pipelineConfig.CdWorkflowRepository pubsubClient *pubsub.PubSubClientServiceImpl - appService app.AppService cdWorkflowService WorkflowService ciPipelineRepository pipelineConfig.CiPipelineRepository materialRepository pipelineConfig.MaterialRepository @@ -157,37 +146,18 @@ type WorkflowDagExecutorImpl struct { devtronAppReleaseContextMap map[int]DevtronAppReleaseContextType devtronAppReleaseContextMapLock *sync.Mutex - deploymentTemplateHistoryService history2.DeploymentTemplateHistoryService - configMapHistoryService history2.ConfigMapHistoryService - pipelineStrategyHistoryService history2.PipelineStrategyHistoryService - appCrudOperationService app.AppCrudOperationService - pipelineConfigRepository chartConfig.PipelineConfigRepository - dockerRegistryIpsConfigService dockerRegistry.DockerRegistryIpsConfigService - chartRepository chartRepoRepository.ChartRepository - strategyHistoryRepository repository3.PipelineStrategyHistoryRepository - deploymentTemplateHistoryRepository repository3.DeploymentTemplateHistoryRepository - configMapRepository chartConfig.ConfigMapRepository - configMapHistoryRepository repository3.ConfigMapHistoryRepository - helmAppService client2.HelmAppService - //TODO fix me next - environmentConfigRepository chartConfig.EnvConfigOverrideRepository - mergeUtil *util.MergeUtil - acdClient application2.ServiceClient - customTagService CustomTagService - deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService - chartRefService chartRef.ChartRefService - imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService + helmAppService client2.HelmAppService + customTagService CustomTagService + imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService cdWorkflowCommonService cd.CdWorkflowCommonService cdTriggerService devtronApps.TriggerService -} -const kedaAutoscaling = "kedaAutoscaling" -const horizontalPodAutoscaler = "HorizontalPodAutoscaler" -const fullnameOverride = "fullnameOverride" -const nameOverride = "nameOverride" -const enabled = "enabled" -const replicaCount = "replicaCount" + deployedConfigurationHistoryService history2.DeployedConfigurationHistoryService + workflowEventPublishService out.WorkflowEventPublishService + manifestCreationService manifest.ManifestCreationService + deploymentTemplateService deploymentTemplate.DeploymentTemplateService +} const ( GIT_COMMIT_HASH_PREFIX = "GIT_COMMIT_HASH" @@ -231,7 +201,6 @@ type TriggerContext struct { func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pipelineConfig.PipelineRepository, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, pubsubClient *pubsub.PubSubClientServiceImpl, - appService app.AppService, cdWorkflowService WorkflowService, ciArtifactRepository repository.CiArtifactRepository, ciPipelineRepository pipelineConfig.CiPipelineRepository, @@ -255,34 +224,20 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi globalPluginService plugin.GlobalPluginService, pluginInputVariableParser PluginInputVariableParser, scopedVariableManager variables.ScopedVariableCMCSManager, - - deploymentTemplateHistoryService history2.DeploymentTemplateHistoryService, - configMapHistoryService history2.ConfigMapHistoryService, - pipelineStrategyHistoryService history2.PipelineStrategyHistoryService, - appCrudOperationService app.AppCrudOperationService, - pipelineConfigRepository chartConfig.PipelineConfigRepository, - dockerRegistryIpsConfigService dockerRegistry.DockerRegistryIpsConfigService, - chartRepository chartRepoRepository.ChartRepository, - strategyHistoryRepository repository3.PipelineStrategyHistoryRepository, - deploymentTemplateHistoryRepository repository3.DeploymentTemplateHistoryRepository, - configMapRepository chartConfig.ConfigMapRepository, - configMapHistoryRepository repository3.ConfigMapHistoryRepository, helmAppService client2.HelmAppService, - environmentConfigRepository chartConfig.EnvConfigOverrideRepository, - mergeUtil *util.MergeUtil, - acdClient application2.ServiceClient, pipelineConfigListenerService PipelineConfigListenerService, customTagService CustomTagService, - deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService, - chartRefService chartRef.ChartRefService, imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService, cdWorkflowCommonService cd.CdWorkflowCommonService, - cdTriggerService devtronApps.TriggerService) *WorkflowDagExecutorImpl { + cdTriggerService devtronApps.TriggerService, + deployedConfigurationHistoryService history2.DeployedConfigurationHistoryService, + workflowEventPublishService out.WorkflowEventPublishService, + manifestCreationService manifest.ManifestCreationService, + deploymentTemplateService deploymentTemplate.DeploymentTemplateService) *WorkflowDagExecutorImpl { wde := &WorkflowDagExecutorImpl{logger: Logger, pipelineRepository: pipelineRepository, cdWorkflowRepository: cdWorkflowRepository, pubsubClient: pubsubClient, - appService: appService, cdWorkflowService: cdWorkflowService, ciPipelineRepository: ciPipelineRepository, ciArtifactRepository: ciArtifactRepository, @@ -311,32 +266,19 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi globalPluginService: globalPluginService, pluginInputVariableParser: pluginInputVariableParser, - devtronAsyncHelmInstallRequestMap: make(map[int]bool), - devtronAsyncHelmInstallRequestLock: &sync.Mutex{}, - devtronAppReleaseContextMap: make(map[int]DevtronAppReleaseContextType), - devtronAppReleaseContextMapLock: &sync.Mutex{}, - - deploymentTemplateHistoryService: deploymentTemplateHistoryService, - configMapHistoryService: configMapHistoryService, - pipelineStrategyHistoryService: pipelineStrategyHistoryService, - appCrudOperationService: appCrudOperationService, - pipelineConfigRepository: pipelineConfigRepository, - dockerRegistryIpsConfigService: dockerRegistryIpsConfigService, - chartRepository: chartRepository, - strategyHistoryRepository: strategyHistoryRepository, - deploymentTemplateHistoryRepository: deploymentTemplateHistoryRepository, - configMapRepository: configMapRepository, - configMapHistoryRepository: configMapHistoryRepository, + devtronAsyncHelmInstallRequestMap: make(map[int]bool), + devtronAsyncHelmInstallRequestLock: &sync.Mutex{}, + devtronAppReleaseContextMap: make(map[int]DevtronAppReleaseContextType), + devtronAppReleaseContextMapLock: &sync.Mutex{}, helmAppService: helmAppService, - environmentConfigRepository: environmentConfigRepository, - mergeUtil: mergeUtil, - acdClient: acdClient, customTagService: customTagService, - deployedAppMetricsService: deployedAppMetricsService, - chartRefService: chartRefService, imageDigestPolicyService: imageDigestPolicyService, cdWorkflowCommonService: cdWorkflowCommonService, cdTriggerService: cdTriggerService, + deployedConfigurationHistoryService: deployedConfigurationHistoryService, + workflowEventPublishService: workflowEventPublishService, + manifestCreationService: manifestCreationService, + deploymentTemplateService: deploymentTemplateService, } config, err := types.GetCdConfig() if err != nil { @@ -2056,7 +1998,7 @@ func (impl *WorkflowDagExecutorImpl) TriggerDeployment(request TriggerRequest) e return releaseErr } //skip updatePreviousDeploymentStatus if Async Install is enabled; handled inside SubscribeDevtronAsyncHelmInstallRequest - if !impl.appService.IsDevtronAsyncInstallModeEnabled(pipeline.DeploymentAppType) { + if !impl.cdTriggerService.IsDevtronAsyncInstallModeEnabled(pipeline.DeploymentAppType) { err1 := impl.updatePreviousDeploymentStatus(runner, pipeline.Id, triggeredAt, triggeredBy) if err1 != nil { impl.logger.Errorw("error while update previous cd workflow runners", "err", err, "runner", runner, "pipelineId", pipeline.Id) @@ -2422,7 +2364,7 @@ func (impl *WorkflowDagExecutorImpl) ManualCdTrigger(triggerContext TriggerConte } // skip updatePreviousDeploymentStatus if Async Install is enabled; handled inside SubscribeDevtronAsyncHelmInstallRequest - if !impl.appService.IsDevtronAsyncInstallModeEnabled(cdPipeline.DeploymentAppType) { + if !impl.cdTriggerService.IsDevtronAsyncInstallModeEnabled(cdPipeline.DeploymentAppType) { // Update previous deployment runner status (in transaction): Failed _, span = otel.Tracer("orchestrator").Start(ctx, "updatePreviousDeploymentStatus") err1 := impl.updatePreviousDeploymentStatus(runner, cdPipeline.Id, triggeredAt, overrideRequest.UserId) @@ -2575,15 +2517,15 @@ func (impl *WorkflowDagExecutorImpl) MarkCurrentDeploymentFailed(runner *pipelin } func (impl *WorkflowDagExecutorImpl) HandleCDTriggerRelease(overrideRequest *bean.ValuesOverrideRequest, ctx context.Context, triggeredAt time.Time, deployedBy int32) (releaseNo int, manifest []byte, err error) { - if impl.appService.IsDevtronAsyncInstallModeEnabled(overrideRequest.DeploymentAppType) { + if impl.cdTriggerService.IsDevtronAsyncInstallModeEnabled(overrideRequest.DeploymentAppType) { // asynchronous mode of installation starts - return impl.TriggerHelmAsyncRelease(overrideRequest, ctx, triggeredAt, deployedBy) + return impl.workflowEventPublishService.TriggerHelmAsyncRelease(overrideRequest, ctx, triggeredAt, deployedBy) } // synchronous mode of installation starts valuesOverrideResponse, builtChartPath, err := impl.BuildManifestForTrigger(overrideRequest, triggeredAt, ctx) _, span := otel.Tracer("orchestrator").Start(ctx, "CreateHistoriesForDeploymentTrigger") - err1 := impl.CreateHistoriesForDeploymentTrigger(valuesOverrideResponse.Pipeline, valuesOverrideResponse.PipelineStrategy, valuesOverrideResponse.EnvOverride, triggeredAt, deployedBy) + err1 := impl.deployedConfigurationHistoryService.CreateHistoriesForDeploymentTrigger(valuesOverrideResponse.Pipeline, valuesOverrideResponse.PipelineStrategy, valuesOverrideResponse.EnvOverride, triggeredAt, deployedBy) if err1 != nil { impl.logger.Errorw("error in saving histories for trigger", "err", err1, "pipelineId", valuesOverrideResponse.Pipeline.Id, "wfrId", overrideRequest.WfrId) } @@ -2595,59 +2537,6 @@ func (impl *WorkflowDagExecutorImpl) HandleCDTriggerRelease(overrideRequest *bea return impl.cdTriggerService.TriggerRelease(overrideRequest, valuesOverrideResponse, builtChartPath, ctx, triggeredAt, deployedBy) } -// TriggerHelmAsyncRelease will publish async helm Install/Upgrade request event for Devtron App releases -func (impl *WorkflowDagExecutorImpl) TriggerHelmAsyncRelease(overrideRequest *bean.ValuesOverrideRequest, ctx context.Context, triggeredAt time.Time, triggeredBy int32) (releaseNo int, manifest []byte, err error) { - // build merged values and save PCO history for the release - valuesOverrideResponse, err := impl.GetValuesOverrideForTrigger(overrideRequest, triggeredAt, ctx) - _, span := otel.Tracer("orchestrator").Start(ctx, "CreateHistoriesForDeploymentTrigger") - // save triggered deployment history - err1 := impl.CreateHistoriesForDeploymentTrigger(valuesOverrideResponse.Pipeline, valuesOverrideResponse.PipelineStrategy, valuesOverrideResponse.EnvOverride, triggeredAt, triggeredBy) - if err1 != nil { - impl.logger.Errorw("error in saving histories for trigger", "err", err1, "pipelineId", valuesOverrideResponse.Pipeline.Id, "wfrId", overrideRequest.WfrId) - } - span.End() - if err != nil { - impl.logger.Errorw("error in fetching values for trigger", "err", err) - return releaseNo, manifest, err - } - - event := &bean7.AsyncCdDeployEvent{ - ValuesOverrideRequest: overrideRequest, - TriggeredAt: triggeredAt, - TriggeredBy: triggeredBy, - } - payload, err := json.Marshal(event) - if err != nil { - impl.logger.Errorw("failed to marshal helm async CD deploy event request", "request", event, "err", err) - return 0, manifest, err - } - - // publish nats event for async installation - err = impl.pubsubClient.Publish(pubsub.DEVTRON_CHART_INSTALL_TOPIC, string(payload)) - if err != nil { - impl.logger.Errorw("failed to publish trigger request event", "topic", pubsub.DEVTRON_CHART_INSTALL_TOPIC, "payload", payload, "err", err) - //update workflow runner status, used in app workflow view - err1 = impl.cdWorkflowCommonService.UpdateCDWorkflowRunnerStatus(ctx, overrideRequest, triggeredAt, pipelineConfig.WorkflowFailed, err.Error()) - if err1 != nil { - impl.logger.Errorw("error in updating the workflow runner status, TriggerHelmAsyncRelease", "err", err1) - } - return 0, manifest, err - } - - //update workflow runner status, used in app workflow view - err = impl.cdWorkflowCommonService.UpdateCDWorkflowRunnerStatus(ctx, overrideRequest, triggeredAt, pipelineConfig.WorkflowInQueue, "") - if err != nil { - impl.logger.Errorw("error in updating the workflow runner status, TriggerHelmAsyncRelease", "err", err) - return 0, manifest, err - } - err = impl.UpdatePreviousQueuedRunnerStatus(overrideRequest.WfrId, overrideRequest.PipelineId, triggeredBy) - if err != nil { - impl.logger.Errorw("error in updating the previous queued workflow runner status, TriggerHelmAsyncRelease", "err", err) - return 0, manifest, err - } - return 0, manifest, nil -} - func (impl *WorkflowDagExecutorImpl) TriggerCD(artifact *repository.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, triggeredAt time.Time) error { impl.logger.Debugw("automatic pipeline trigger attempt async", "artifactId", artifact.Id) @@ -2725,14 +2614,13 @@ func (impl *WorkflowDagExecutorImpl) SetPipelineFieldsInOverrideRequest(override // write integration/unit test for each function func (impl *WorkflowDagExecutorImpl) BuildManifestForTrigger(overrideRequest *bean.ValuesOverrideRequest, triggeredAt time.Time, ctx context.Context) (valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, err error) { - valuesOverrideResponse = &app.ValuesOverrideResponse{} - valuesOverrideResponse, err = impl.GetValuesOverrideForTrigger(overrideRequest, triggeredAt, ctx) + valuesOverrideResponse, err = impl.manifestCreationService.GetValuesOverrideForTrigger(overrideRequest, triggeredAt, ctx) if err != nil { impl.logger.Errorw("error in fetching values for trigger", "err", err) return valuesOverrideResponse, "", err } - builtChartPath, err = impl.appService.BuildChartAndGetPath(overrideRequest.AppName, valuesOverrideResponse.EnvOverride, ctx) + builtChartPath, err = impl.deploymentTemplateService.BuildChartAndGetPath(overrideRequest.AppName, valuesOverrideResponse.EnvOverride, ctx) if err != nil { impl.logger.Errorw("error in parsing reference chart", "err", err) return valuesOverrideResponse, "", err @@ -2740,787 +2628,6 @@ func (impl *WorkflowDagExecutorImpl) BuildManifestForTrigger(overrideRequest *be return valuesOverrideResponse, builtChartPath, err } -func (impl *WorkflowDagExecutorImpl) CreateHistoriesForDeploymentTrigger(pipeline *pipelineConfig.Pipeline, strategy *chartConfig.PipelineStrategy, envOverride *chartConfig.EnvConfigOverride, deployedOn time.Time, deployedBy int32) error { - //creating history for deployment template - deploymentTemplateHistory, err := impl.deploymentTemplateHistoryService.CreateDeploymentTemplateHistoryForDeploymentTrigger(pipeline, envOverride, envOverride.Chart.ImageDescriptorTemplate, deployedOn, deployedBy) - if err != nil { - impl.logger.Errorw("error in creating deployment template history for deployment trigger", "err", err) - return err - } - cmId, csId, err := impl.configMapHistoryService.CreateCMCSHistoryForDeploymentTrigger(pipeline, deployedOn, deployedBy) - if err != nil { - impl.logger.Errorw("error in creating CM/CS history for deployment trigger", "err", err) - return err - } - if strategy != nil { - err = impl.pipelineStrategyHistoryService.CreateStrategyHistoryForDeploymentTrigger(strategy, deployedOn, deployedBy, pipeline.TriggerType) - if err != nil { - impl.logger.Errorw("error in creating strategy history for deployment trigger", "err", err) - return err - } - } - - var variableSnapshotHistories = util4.GetBeansPtr( - repository5.GetSnapshotBean(deploymentTemplateHistory.Id, repository5.HistoryReferenceTypeDeploymentTemplate, envOverride.VariableSnapshot), - repository5.GetSnapshotBean(cmId, repository5.HistoryReferenceTypeConfigMap, envOverride.VariableSnapshotForCM), - repository5.GetSnapshotBean(csId, repository5.HistoryReferenceTypeSecret, envOverride.VariableSnapshotForCS), - ) - if len(variableSnapshotHistories) > 0 { - err = impl.scopedVariableManager.SaveVariableHistoriesForTrigger(variableSnapshotHistories, deployedBy) - if err != nil { - return err - } - } - return nil -} - -func (impl *WorkflowDagExecutorImpl) GetValuesOverrideForTrigger(overrideRequest *bean.ValuesOverrideRequest, triggeredAt time.Time, ctx context.Context) (*app.ValuesOverrideResponse, error) { - if overrideRequest.DeploymentType == models.DEPLOYMENTTYPE_UNKNOWN { - overrideRequest.DeploymentType = models.DEPLOYMENTTYPE_DEPLOY - } - if len(overrideRequest.DeploymentWithConfig) == 0 { - overrideRequest.DeploymentWithConfig = bean.DEPLOYMENT_CONFIG_TYPE_LAST_SAVED - } - valuesOverrideResponse := &app.ValuesOverrideResponse{} - isPipelineOverrideCreated := overrideRequest.PipelineOverrideId > 0 - pipeline, err := impl.pipelineRepository.FindById(overrideRequest.PipelineId) - valuesOverrideResponse.Pipeline = pipeline - if err != nil { - impl.logger.Errorw("error in fetching pipeline by pipeline id", "err", err, "pipeline-id-", overrideRequest.PipelineId) - return valuesOverrideResponse, err - } - - _, span := otel.Tracer("orchestrator").Start(ctx, "ciArtifactRepository.Get") - artifact, err := impl.ciArtifactRepository.Get(overrideRequest.CiArtifactId) - valuesOverrideResponse.Artifact = artifact - span.End() - if err != nil { - return valuesOverrideResponse, err - } - overrideRequest.Image = artifact.Image - - strategy, err := impl.GetDeploymentStrategyByTriggerType(overrideRequest, ctx) - valuesOverrideResponse.PipelineStrategy = strategy - if err != nil { - impl.logger.Errorw("error in getting strategy by trigger type", "err", err) - return valuesOverrideResponse, err - } - - envOverride, err := impl.GetEnvOverrideByTriggerType(overrideRequest, triggeredAt, ctx) - valuesOverrideResponse.EnvOverride = envOverride - if err != nil { - impl.logger.Errorw("error in getting env override by trigger type", "err", err) - return valuesOverrideResponse, err - } - appMetrics, err := impl.GetAppMetricsByTriggerType(overrideRequest, ctx) - valuesOverrideResponse.AppMetrics = appMetrics - if err != nil { - impl.logger.Errorw("error in getting app metrics by trigger type", "err", err) - return valuesOverrideResponse, err - } - var ( - pipelineOverride *chartConfig.PipelineOverride - configMapJson, appLabelJsonByte []byte - ) - - // Conditional Block based on PipelineOverrideCreated --> start - if !isPipelineOverrideCreated { - _, span = otel.Tracer("orchestrator").Start(ctx, "savePipelineOverride") - pipelineOverride, err = impl.savePipelineOverride(overrideRequest, envOverride.Id, triggeredAt) - span.End() - if err != nil { - return valuesOverrideResponse, err - } - overrideRequest.PipelineOverrideId = pipelineOverride.Id - } else { - pipelineOverride, err = impl.pipelineOverrideRepository.FindById(overrideRequest.PipelineOverrideId) - if err != nil { - impl.logger.Errorw("error in getting pipelineOverride for valuesOverrideResponse", "PipelineOverrideId", overrideRequest.PipelineOverrideId) - return nil, err - } - } - // Conditional Block based on PipelineOverrideCreated --> end - valuesOverrideResponse.PipelineOverride = pipelineOverride - - //TODO: check status and apply lock - releaseOverrideJson, err := impl.getReleaseOverride(envOverride, overrideRequest, artifact, pipelineOverride, strategy, &appMetrics) - valuesOverrideResponse.ReleaseOverrideJSON = releaseOverrideJson - if err != nil { - return valuesOverrideResponse, err - } - - // Conditional Block based on PipelineOverrideCreated --> start - if !isPipelineOverrideCreated { - chartVersion := envOverride.Chart.ChartVersion - _, span = otel.Tracer("orchestrator").Start(ctx, "getConfigMapAndSecretJsonV2") - scope := getScopeForVariables(overrideRequest, envOverride) - request := createConfigMapAndSecretJsonRequest(overrideRequest, envOverride, chartVersion, scope) - - configMapJson, err = impl.getConfigMapAndSecretJsonV2(request, envOverride) - span.End() - if err != nil { - impl.logger.Errorw("error in fetching config map n secret ", "err", err) - configMapJson = nil - } - _, span = otel.Tracer("orchestrator").Start(ctx, "appCrudOperationService.GetLabelsByAppIdForDeployment") - appLabelJsonByte, err = impl.appCrudOperationService.GetLabelsByAppIdForDeployment(overrideRequest.AppId) - span.End() - if err != nil { - impl.logger.Errorw("error in fetching app labels for gitOps commit", "err", err) - appLabelJsonByte = nil - } - - mergedValues, err := impl.mergeOverrideValues(envOverride, releaseOverrideJson, configMapJson, appLabelJsonByte, strategy) - appName := fmt.Sprintf("%s-%s", overrideRequest.AppName, envOverride.Environment.Name) - mergedValues = impl.autoscalingCheckBeforeTrigger(ctx, appName, envOverride.Namespace, mergedValues, overrideRequest) - - _, span = otel.Tracer("orchestrator").Start(ctx, "dockerRegistryIpsConfigService.HandleImagePullSecretOnApplicationDeployment") - // handle image pull secret if access given - mergedValues, err = impl.dockerRegistryIpsConfigService.HandleImagePullSecretOnApplicationDeployment(envOverride.Environment, artifact, pipeline.CiPipelineId, mergedValues) - span.End() - if err != nil { - return valuesOverrideResponse, err - } - - pipelineOverride.PipelineMergedValues = string(mergedValues) - valuesOverrideResponse.MergedValues = string(mergedValues) - err = impl.pipelineOverrideRepository.Update(pipelineOverride) - if err != nil { - return valuesOverrideResponse, err - } - valuesOverrideResponse.PipelineOverride = pipelineOverride - } else { - valuesOverrideResponse.MergedValues = pipelineOverride.PipelineMergedValues - } - // Conditional Block based on PipelineOverrideCreated --> end - return valuesOverrideResponse, err -} - -func createConfigMapAndSecretJsonRequest(overrideRequest *bean.ValuesOverrideRequest, envOverride *chartConfig.EnvConfigOverride, chartVersion string, scope resourceQualifiers.Scope) ConfigMapAndSecretJsonV2 { - request := ConfigMapAndSecretJsonV2{ - AppId: overrideRequest.AppId, - EnvId: envOverride.TargetEnvironment, - PipeLineId: overrideRequest.PipelineId, - ChartVersion: chartVersion, - DeploymentWithConfig: overrideRequest.DeploymentWithConfig, - wfrIdForDeploymentWithSpecificTrigger: overrideRequest.WfrIdForDeploymentWithSpecificTrigger, - Scope: scope, - } - return request -} - -func getScopeForVariables(overrideRequest *bean.ValuesOverrideRequest, envOverride *chartConfig.EnvConfigOverride) resourceQualifiers.Scope { - scope := resourceQualifiers.Scope{ - AppId: overrideRequest.AppId, - EnvId: envOverride.TargetEnvironment, - ClusterId: envOverride.Environment.Id, - SystemMetadata: &resourceQualifiers.SystemMetadata{ - EnvironmentName: envOverride.Environment.Name, - ClusterName: envOverride.Environment.Cluster.ClusterName, - Namespace: envOverride.Environment.Namespace, - ImageTag: util3.GetImageTagFromImage(overrideRequest.Image), - AppName: overrideRequest.AppName, - Image: overrideRequest.Image, - }, - } - return scope -} - -func (impl *WorkflowDagExecutorImpl) GetDeploymentStrategyByTriggerType(overrideRequest *bean.ValuesOverrideRequest, ctx context.Context) (*chartConfig.PipelineStrategy, error) { - - strategy := &chartConfig.PipelineStrategy{} - var err error - if overrideRequest.DeploymentWithConfig == bean.DEPLOYMENT_CONFIG_TYPE_SPECIFIC_TRIGGER { - _, span := otel.Tracer("orchestrator").Start(ctx, "strategyHistoryRepository.GetHistoryByPipelineIdAndWfrId") - strategyHistory, err := impl.strategyHistoryRepository.GetHistoryByPipelineIdAndWfrId(overrideRequest.PipelineId, overrideRequest.WfrIdForDeploymentWithSpecificTrigger) - span.End() - if err != nil { - impl.logger.Errorw("error in getting deployed strategy history by pipleinId and wfrId", "err", err, "pipelineId", overrideRequest.PipelineId, "wfrId", overrideRequest.WfrIdForDeploymentWithSpecificTrigger) - return nil, err - } - strategy.Strategy = strategyHistory.Strategy - strategy.Config = strategyHistory.Config - strategy.PipelineId = overrideRequest.PipelineId - } else if overrideRequest.DeploymentWithConfig == bean.DEPLOYMENT_CONFIG_TYPE_LAST_SAVED { - if overrideRequest.ForceTrigger { - _, span := otel.Tracer("orchestrator").Start(ctx, "pipelineConfigRepository.GetDefaultStrategyByPipelineId") - strategy, err = impl.pipelineConfigRepository.GetDefaultStrategyByPipelineId(overrideRequest.PipelineId) - span.End() - } else { - var deploymentTemplate chartRepoRepository.DeploymentStrategy - if overrideRequest.DeploymentTemplate == "ROLLING" { - deploymentTemplate = chartRepoRepository.DEPLOYMENT_STRATEGY_ROLLING - } else if overrideRequest.DeploymentTemplate == "BLUE-GREEN" { - deploymentTemplate = chartRepoRepository.DEPLOYMENT_STRATEGY_BLUE_GREEN - } else if overrideRequest.DeploymentTemplate == "CANARY" { - deploymentTemplate = chartRepoRepository.DEPLOYMENT_STRATEGY_CANARY - } else if overrideRequest.DeploymentTemplate == "RECREATE" { - deploymentTemplate = chartRepoRepository.DEPLOYMENT_STRATEGY_RECREATE - } - - if len(deploymentTemplate) > 0 { - _, span := otel.Tracer("orchestrator").Start(ctx, "pipelineConfigRepository.FindByStrategyAndPipelineId") - strategy, err = impl.pipelineConfigRepository.FindByStrategyAndPipelineId(deploymentTemplate, overrideRequest.PipelineId) - span.End() - } else { - _, span := otel.Tracer("orchestrator").Start(ctx, "pipelineConfigRepository.GetDefaultStrategyByPipelineId") - strategy, err = impl.pipelineConfigRepository.GetDefaultStrategyByPipelineId(overrideRequest.PipelineId) - span.End() - } - } - if err != nil && errors2.IsNotFound(err) == false { - impl.logger.Errorf("invalid state", "err", err, "req", strategy) - return nil, err - } - } - return strategy, nil -} - -func (impl *WorkflowDagExecutorImpl) GetEnvOverrideByTriggerType(overrideRequest *bean.ValuesOverrideRequest, triggeredAt time.Time, ctx context.Context) (*chartConfig.EnvConfigOverride, error) { - - envOverride := &chartConfig.EnvConfigOverride{} - - var err error - if overrideRequest.DeploymentWithConfig == bean.DEPLOYMENT_CONFIG_TYPE_SPECIFIC_TRIGGER { - _, span := otel.Tracer("orchestrator").Start(ctx, "deploymentTemplateHistoryRepository.GetHistoryByPipelineIdAndWfrId") - deploymentTemplateHistory, err := impl.deploymentTemplateHistoryRepository.GetHistoryByPipelineIdAndWfrId(overrideRequest.PipelineId, overrideRequest.WfrIdForDeploymentWithSpecificTrigger) - //VARIABLE_SNAPSHOT_GET and resolve - - span.End() - if err != nil { - impl.logger.Errorw("error in getting deployed deployment template history by pipelineId and wfrId", "err", err, "pipelineId", &overrideRequest, "wfrId", overrideRequest.WfrIdForDeploymentWithSpecificTrigger) - return nil, err - } - templateName := deploymentTemplateHistory.TemplateName - templateVersion := deploymentTemplateHistory.TemplateVersion - if templateName == "Rollout Deployment" { - templateName = "" - } - //getting chart_ref by id - _, span = otel.Tracer("orchestrator").Start(ctx, "chartRefRepository.FindByVersionAndName") - chartRefDto, err := impl.chartRefService.FindByVersionAndName(templateName, templateVersion) - span.End() - if err != nil { - impl.logger.Errorw("error in getting chartRef by version and name", "err", err, "version", templateVersion, "name", templateName) - return nil, err - } - //assuming that if a chartVersion is deployed then it's envConfigOverride will be available - _, span = otel.Tracer("orchestrator").Start(ctx, "environmentConfigRepository.GetByAppIdEnvIdAndChartRefId") - envOverride, err = impl.environmentConfigRepository.GetByAppIdEnvIdAndChartRefId(overrideRequest.AppId, overrideRequest.EnvId, chartRefDto.Id) - span.End() - if err != nil { - impl.logger.Errorw("error in getting envConfigOverride for pipeline for specific chartVersion", "err", err, "appId", overrideRequest.AppId, "envId", overrideRequest.EnvId, "chartRefId", chartRefDto.Id) - return nil, err - } - - _, span = otel.Tracer("orchestrator").Start(ctx, "envRepository.FindById") - env, err := impl.envRepository.FindById(envOverride.TargetEnvironment) - span.End() - if err != nil { - impl.logger.Errorw("unable to find env", "err", err) - return nil, err - } - envOverride.Environment = env - - //updating historical data in envConfigOverride and appMetrics flag - envOverride.IsOverride = true - envOverride.EnvOverrideValues = deploymentTemplateHistory.Template - reference := repository5.HistoryReference{ - HistoryReferenceId: deploymentTemplateHistory.Id, - HistoryReferenceType: repository5.HistoryReferenceTypeDeploymentTemplate, - } - variableMap, resolvedTemplate, err := impl.scopedVariableManager.GetVariableSnapshotAndResolveTemplate(envOverride.EnvOverrideValues, parsers.JsonVariableTemplate, reference, true, false) - envOverride.ResolvedEnvOverrideValues = resolvedTemplate - envOverride.VariableSnapshot = variableMap - if err != nil { - return envOverride, err - } - } else if overrideRequest.DeploymentWithConfig == bean.DEPLOYMENT_CONFIG_TYPE_LAST_SAVED { - _, span := otel.Tracer("orchestrator").Start(ctx, "environmentConfigRepository.ActiveEnvConfigOverride") - envOverride, err = impl.environmentConfigRepository.ActiveEnvConfigOverride(overrideRequest.AppId, overrideRequest.EnvId) - - var chart *chartRepoRepository.Chart - span.End() - if err != nil { - impl.logger.Errorw("invalid state", "err", err, "req", overrideRequest) - return nil, err - } - if envOverride.Id == 0 { - _, span = otel.Tracer("orchestrator").Start(ctx, "chartRepository.FindLatestChartForAppByAppId") - chart, err = impl.chartRepository.FindLatestChartForAppByAppId(overrideRequest.AppId) - span.End() - if err != nil { - impl.logger.Errorw("invalid state", "err", err, "req", overrideRequest) - return nil, err - } - _, span = otel.Tracer("orchestrator").Start(ctx, "environmentConfigRepository.FindChartByAppIdAndEnvIdAndChartRefId") - envOverride, err = impl.environmentConfigRepository.FindChartByAppIdAndEnvIdAndChartRefId(overrideRequest.AppId, overrideRequest.EnvId, chart.ChartRefId) - span.End() - if err != nil && !errors2.IsNotFound(err) { - impl.logger.Errorw("invalid state", "err", err, "req", overrideRequest) - return nil, err - } - - //creating new env override config - if errors2.IsNotFound(err) || envOverride == nil { - _, span = otel.Tracer("orchestrator").Start(ctx, "envRepository.FindById") - environment, err := impl.envRepository.FindById(overrideRequest.EnvId) - span.End() - if err != nil && !util.IsErrNoRows(err) { - return nil, err - } - envOverride = &chartConfig.EnvConfigOverride{ - Active: true, - ManualReviewed: true, - Status: models.CHARTSTATUS_SUCCESS, - TargetEnvironment: overrideRequest.EnvId, - ChartId: chart.Id, - AuditLog: sql.AuditLog{UpdatedBy: overrideRequest.UserId, UpdatedOn: triggeredAt, CreatedOn: triggeredAt, CreatedBy: overrideRequest.UserId}, - Namespace: environment.Namespace, - IsOverride: false, - EnvOverrideValues: "{}", - Latest: false, - IsBasicViewLocked: chart.IsBasicViewLocked, - CurrentViewEditor: chart.CurrentViewEditor, - } - _, span = otel.Tracer("orchestrator").Start(ctx, "environmentConfigRepository.Save") - err = impl.environmentConfigRepository.Save(envOverride) - span.End() - if err != nil { - impl.logger.Errorw("error in creating envconfig", "data", envOverride, "error", err) - return nil, err - } - } - envOverride.Chart = chart - } else if envOverride.Id > 0 && !envOverride.IsOverride { - _, span = otel.Tracer("orchestrator").Start(ctx, "chartRepository.FindLatestChartForAppByAppId") - chart, err = impl.chartRepository.FindLatestChartForAppByAppId(overrideRequest.AppId) - span.End() - if err != nil { - impl.logger.Errorw("invalid state", "err", err, "req", overrideRequest) - return nil, err - } - envOverride.Chart = chart - } - - _, span = otel.Tracer("orchestrator").Start(ctx, "envRepository.FindById") - env, err := impl.envRepository.FindById(envOverride.TargetEnvironment) - span.End() - if err != nil { - impl.logger.Errorw("unable to find env", "err", err) - return nil, err - } - envOverride.Environment = env - scope := getScopeForVariables(overrideRequest, envOverride) - if envOverride.IsOverride { - - entity := repository5.GetEntity(envOverride.Id, repository5.EntityTypeDeploymentTemplateEnvLevel) - resolvedTemplate, variableMap, err := impl.scopedVariableManager.GetMappedVariablesAndResolveTemplate(envOverride.EnvOverrideValues, scope, entity, true) - envOverride.ResolvedEnvOverrideValues = resolvedTemplate - envOverride.VariableSnapshot = variableMap - if err != nil { - return envOverride, err - } - - } else { - entity := repository5.GetEntity(chart.Id, repository5.EntityTypeDeploymentTemplateAppLevel) - resolvedTemplate, variableMap, err := impl.scopedVariableManager.GetMappedVariablesAndResolveTemplate(chart.GlobalOverride, scope, entity, true) - envOverride.Chart.ResolvedGlobalOverride = resolvedTemplate - envOverride.VariableSnapshot = variableMap - if err != nil { - return envOverride, err - } - - } - } - - return envOverride, nil -} - -func (impl *WorkflowDagExecutorImpl) GetAppMetricsByTriggerType(overrideRequest *bean.ValuesOverrideRequest, ctx context.Context) (bool, error) { - - var appMetrics bool - if overrideRequest.DeploymentWithConfig == bean.DEPLOYMENT_CONFIG_TYPE_SPECIFIC_TRIGGER { - _, span := otel.Tracer("orchestrator").Start(ctx, "deploymentTemplateHistoryRepository.GetHistoryByPipelineIdAndWfrId") - deploymentTemplateHistory, err := impl.deploymentTemplateHistoryRepository.GetHistoryByPipelineIdAndWfrId(overrideRequest.PipelineId, overrideRequest.WfrIdForDeploymentWithSpecificTrigger) - span.End() - if err != nil { - impl.logger.Errorw("error in getting deployed deployment template history by pipelineId and wfrId", "err", err, "pipelineId", &overrideRequest, "wfrId", overrideRequest.WfrIdForDeploymentWithSpecificTrigger) - return appMetrics, err - } - appMetrics = deploymentTemplateHistory.IsAppMetricsEnabled - - } else if overrideRequest.DeploymentWithConfig == bean.DEPLOYMENT_CONFIG_TYPE_LAST_SAVED { - _, span := otel.Tracer("orchestrator").Start(ctx, "deployedAppMetricsService.GetMetricsFlagForAPipelineByAppIdAndEnvId") - isAppMetricsEnabled, err := impl.deployedAppMetricsService.GetMetricsFlagForAPipelineByAppIdAndEnvId(overrideRequest.AppId, overrideRequest.EnvId) - if err != nil { - impl.logger.Errorw("error, GetMetricsFlagForAPipelineByAppIdAndEnvId", "err", err, "appId", overrideRequest.AppId, "envId", overrideRequest.EnvId) - return appMetrics, err - } - span.End() - appMetrics = isAppMetricsEnabled - } - return appMetrics, nil -} - -type ConfigMapAndSecretJsonV2 struct { - AppId int - EnvId int - PipeLineId int - ChartVersion string - DeploymentWithConfig bean.DeploymentConfigurationType - wfrIdForDeploymentWithSpecificTrigger int - Scope resourceQualifiers.Scope -} - -func (impl *WorkflowDagExecutorImpl) getConfigMapAndSecretJsonV2(request ConfigMapAndSecretJsonV2, envOverride *chartConfig.EnvConfigOverride) ([]byte, error) { - - var configMapJson, secretDataJson, configMapJsonApp, secretDataJsonApp, configMapJsonEnv, secretDataJsonEnv string - - var err error - configMapA := &chartConfig.ConfigMapAppModel{} - configMapE := &chartConfig.ConfigMapEnvModel{} - configMapHistory, secretHistory := &repository3.ConfigmapAndSecretHistory{}, &repository3.ConfigmapAndSecretHistory{} - - merged := []byte("{}") - if request.DeploymentWithConfig == bean.DEPLOYMENT_CONFIG_TYPE_LAST_SAVED { - configMapA, err = impl.configMapRepository.GetByAppIdAppLevel(request.AppId) - if err != nil && pg.ErrNoRows != err { - return []byte("{}"), err - } - if configMapA != nil && configMapA.Id > 0 { - configMapJsonApp = configMapA.ConfigMapData - secretDataJsonApp = configMapA.SecretData - } - - configMapE, err = impl.configMapRepository.GetByAppIdAndEnvIdEnvLevel(request.AppId, request.EnvId) - if err != nil && pg.ErrNoRows != err { - return []byte("{}"), err - } - if configMapE != nil && configMapE.Id > 0 { - configMapJsonEnv = configMapE.ConfigMapData - secretDataJsonEnv = configMapE.SecretData - } - - } else if request.DeploymentWithConfig == bean.DEPLOYMENT_CONFIG_TYPE_SPECIFIC_TRIGGER { - - //fetching history and setting envLevelConfig and not appLevelConfig because history already contains merged appLevel and envLevel configs - configMapHistory, err = impl.configMapHistoryRepository.GetHistoryByPipelineIdAndWfrId(request.PipeLineId, request.wfrIdForDeploymentWithSpecificTrigger, repository3.CONFIGMAP_TYPE) - if err != nil { - impl.logger.Errorw("error in getting config map history config by pipelineId and wfrId ", "err", err, "pipelineId", request.PipeLineId, "wfrid", request.wfrIdForDeploymentWithSpecificTrigger) - return []byte("{}"), err - } - configMapJsonEnv = configMapHistory.Data - - secretHistory, err = impl.configMapHistoryRepository.GetHistoryByPipelineIdAndWfrId(request.PipeLineId, request.wfrIdForDeploymentWithSpecificTrigger, repository3.SECRET_TYPE) - if err != nil { - impl.logger.Errorw("error in getting config map history config by pipelineId and wfrId ", "err", err, "pipelineId", request.PipeLineId, "wfrid", request.wfrIdForDeploymentWithSpecificTrigger) - return []byte("{}"), err - } - secretDataJsonEnv = secretHistory.Data - } - configMapJson, err = impl.mergeUtil.ConfigMapMerge(configMapJsonApp, configMapJsonEnv) - if err != nil { - return []byte("{}"), err - } - chartMajorVersion, chartMinorVersion, err := util4.ExtractChartVersion(request.ChartVersion) - if err != nil { - impl.logger.Errorw("chart version parsing", "err", err) - return []byte("{}"), err - } - secretDataJson, err = impl.mergeUtil.ConfigSecretMerge(secretDataJsonApp, secretDataJsonEnv, chartMajorVersion, chartMinorVersion, false) - if err != nil { - return []byte("{}"), err - } - configResponseR := bean.ConfigMapRootJson{} - configResponse := bean.ConfigMapJson{} - if configMapJson != "" { - err = json.Unmarshal([]byte(configMapJson), &configResponse) - if err != nil { - return []byte("{}"), err - } - } - configResponseR.ConfigMapJson = configResponse - secretResponseR := bean.ConfigSecretRootJson{} - secretResponse := bean.ConfigSecretJson{} - if configMapJson != "" { - err = json.Unmarshal([]byte(secretDataJson), &secretResponse) - if err != nil { - return []byte("{}"), err - } - } - secretResponseR.ConfigSecretJson = secretResponse - - configMapByte, err := json.Marshal(configResponseR) - if err != nil { - return []byte("{}"), err - } - secretDataByte, err := json.Marshal(secretResponseR) - if err != nil { - return []byte("{}"), err - - } - resolvedCM, resolvedCS, snapshotCM, snapshotCS, err := impl.scopedVariableManager.ResolveCMCSTrigger(request.DeploymentWithConfig, request.Scope, configMapA.Id, configMapE.Id, configMapByte, secretDataByte, configMapHistory.Id, secretHistory.Id) - if err != nil { - return []byte("{}"), err - } - envOverride.VariableSnapshotForCM = snapshotCM - envOverride.VariableSnapshotForCS = snapshotCS - - merged, err = impl.mergeUtil.JsonPatch([]byte(resolvedCM), []byte(resolvedCS)) - - if err != nil { - return []byte("{}"), err - } - - return merged, nil -} - -func (impl *WorkflowDagExecutorImpl) savePipelineOverride(overrideRequest *bean.ValuesOverrideRequest, envOverrideId int, triggeredAt time.Time) (override *chartConfig.PipelineOverride, err error) { - currentReleaseNo, err := impl.pipelineOverrideRepository.GetCurrentPipelineReleaseCounter(overrideRequest.PipelineId) - if err != nil { - return nil, err - } - po := &chartConfig.PipelineOverride{ - EnvConfigOverrideId: envOverrideId, - Status: models.CHARTSTATUS_NEW, - PipelineId: overrideRequest.PipelineId, - CiArtifactId: overrideRequest.CiArtifactId, - PipelineReleaseCounter: currentReleaseNo + 1, - CdWorkflowId: overrideRequest.CdWorkflowId, - AuditLog: sql.AuditLog{CreatedBy: overrideRequest.UserId, CreatedOn: triggeredAt, UpdatedOn: triggeredAt, UpdatedBy: overrideRequest.UserId}, - DeploymentType: overrideRequest.DeploymentType, - } - - err = impl.pipelineOverrideRepository.Save(po) - if err != nil { - return nil, err - } - err = impl.checkAndFixDuplicateReleaseNo(po) - if err != nil { - impl.logger.Errorw("error in checking release no duplicacy", "pipeline", po, "err", err) - return nil, err - } - return po, nil -} - -func (impl *WorkflowDagExecutorImpl) getReleaseOverride(envOverride *chartConfig.EnvConfigOverride, overrideRequest *bean.ValuesOverrideRequest, artifact *repository.CiArtifact, pipelineOverride *chartConfig.PipelineOverride, strategy *chartConfig.PipelineStrategy, appMetrics *bool) (releaseOverride string, err error) { - - artifactImage := artifact.Image - imageTag := strings.Split(artifactImage, ":") - - imageTagLen := len(imageTag) - - imageName := "" - - for i := 0; i < imageTagLen-1; i++ { - if i != imageTagLen-2 { - imageName = imageName + imageTag[i] + ":" - } else { - imageName = imageName + imageTag[i] - } - } - - appId := strconv.Itoa(overrideRequest.AppId) - envId := strconv.Itoa(overrideRequest.EnvId) - - deploymentStrategy := "" - if strategy != nil { - deploymentStrategy = string(strategy.Strategy) - } - - digestConfigurationRequest := imageDigestPolicy.DigestPolicyConfigurationRequest{PipelineId: overrideRequest.PipelineId} - digestPolicyConfigurations, err := impl.imageDigestPolicyService.GetDigestPolicyConfigurations(digestConfigurationRequest) - if err != nil { - impl.logger.Errorw("error in checking if isImageDigestPolicyConfiguredForPipeline", "err", err, "pipelineId", overrideRequest.PipelineId) - return "", err - } - - if digestPolicyConfigurations.UseDigestForTrigger() { - imageTag[imageTagLen-1] = fmt.Sprintf("%s@%s", imageTag[imageTagLen-1], artifact.ImageDigest) - } - - releaseAttribute := app.ReleaseAttributes{ - Name: imageName, - Tag: imageTag[imageTagLen-1], - PipelineName: overrideRequest.PipelineName, - ReleaseVersion: strconv.Itoa(pipelineOverride.PipelineReleaseCounter), - DeploymentType: deploymentStrategy, - App: appId, - Env: envId, - AppMetrics: appMetrics, - } - override, err := util4.Tprintf(envOverride.Chart.ImageDescriptorTemplate, releaseAttribute) - if err != nil { - return "", &util.ApiError{InternalMessage: "unable to render ImageDescriptorTemplate"} - } - if overrideRequest.AdditionalOverride != nil { - userOverride, err := overrideRequest.AdditionalOverride.MarshalJSON() - if err != nil { - return "", err - } - data, err := impl.mergeUtil.JsonPatch(userOverride, []byte(override)) - if err != nil { - return "", err - } - override = string(data) - } - return override, nil -} - -func (impl *WorkflowDagExecutorImpl) mergeOverrideValues(envOverride *chartConfig.EnvConfigOverride, - releaseOverrideJson string, - configMapJson []byte, - appLabelJsonByte []byte, - strategy *chartConfig.PipelineStrategy, -) (mergedValues []byte, err error) { - - //merge three values on the fly - //ordering is important here - //global < environment < db< release - var merged []byte - if !envOverride.IsOverride { - merged, err = impl.mergeUtil.JsonPatch([]byte("{}"), []byte(envOverride.Chart.ResolvedGlobalOverride)) - if err != nil { - return nil, err - } - } else { - merged, err = impl.mergeUtil.JsonPatch([]byte("{}"), []byte(envOverride.ResolvedEnvOverrideValues)) - if err != nil { - return nil, err - } - } - if strategy != nil && len(strategy.Config) > 0 { - merged, err = impl.mergeUtil.JsonPatch(merged, []byte(strategy.Config)) - if err != nil { - return nil, err - } - } - merged, err = impl.mergeUtil.JsonPatch(merged, []byte(releaseOverrideJson)) - if err != nil { - return nil, err - } - if configMapJson != nil { - merged, err = impl.mergeUtil.JsonPatch(merged, configMapJson) - if err != nil { - return nil, err - } - } - if appLabelJsonByte != nil { - merged, err = impl.mergeUtil.JsonPatch(merged, appLabelJsonByte) - if err != nil { - return nil, err - } - } - return merged, nil -} - -func (impl *WorkflowDagExecutorImpl) autoscalingCheckBeforeTrigger(ctx context.Context, appName string, namespace string, merged []byte, overrideRequest *bean.ValuesOverrideRequest) []byte { - //pipeline := overrideRequest.Pipeline - var appId = overrideRequest.AppId - pipelineId := overrideRequest.PipelineId - var appDeploymentType = overrideRequest.DeploymentAppType - var clusterId = overrideRequest.ClusterId - deploymentType := overrideRequest.DeploymentType - templateMap := make(map[string]interface{}) - err := json.Unmarshal(merged, &templateMap) - if err != nil { - return merged - } - - hpaResourceRequest := impl.getAutoScalingReplicaCount(templateMap, appName) - impl.logger.Debugw("autoscalingCheckBeforeTrigger", "hpaResourceRequest", hpaResourceRequest) - if hpaResourceRequest.IsEnable { - resourceManifest := make(map[string]interface{}) - if util.IsAcdApp(appDeploymentType) { - query := &application3.ApplicationResourceRequest{ - Name: &appName, - Version: &hpaResourceRequest.Version, - Group: &hpaResourceRequest.Group, - Kind: &hpaResourceRequest.Kind, - ResourceName: &hpaResourceRequest.ResourceName, - Namespace: &namespace, - } - recv, err := impl.acdClient.GetResource(ctx, query) - impl.logger.Debugw("resource manifest get replica count", "response", recv) - if err != nil { - impl.logger.Errorw("ACD Get Resource API Failed", "err", err) - middleware.AcdGetResourceCounter.WithLabelValues(strconv.Itoa(appId), namespace, appName).Inc() - return merged - } - if recv != nil && len(*recv.Manifest) > 0 { - err := json.Unmarshal([]byte(*recv.Manifest), &resourceManifest) - if err != nil { - impl.logger.Errorw("unmarshal failed for hpa check", "err", err) - return merged - } - } - } else { - version := "v2beta2" - k8sResource, err := impl.k8sCommonService.GetResource(ctx, &k8s.ResourceRequestBean{ClusterId: clusterId, - K8sRequest: &util5.K8sRequestBean{ResourceIdentifier: util5.ResourceIdentifier{Name: hpaResourceRequest.ResourceName, - Namespace: namespace, GroupVersionKind: schema.GroupVersionKind{Group: hpaResourceRequest.Group, Kind: hpaResourceRequest.Kind, Version: version}}}}) - if err != nil { - impl.logger.Errorw("error occurred while fetching resource for app", "resourceName", hpaResourceRequest.ResourceName, "err", err) - return merged - } - resourceManifest = k8sResource.ManifestResponse.Manifest.Object - } - if len(resourceManifest) > 0 { - statusMap := resourceManifest["status"].(map[string]interface{}) - currentReplicaVal := statusMap["currentReplicas"] - currentReplicaCount, err := util4.ParseFloatNumber(currentReplicaVal) - if err != nil { - impl.logger.Errorw("error occurred while parsing replica count", "currentReplicas", currentReplicaVal, "err", err) - return merged - } - - reqReplicaCount := impl.fetchRequiredReplicaCount(currentReplicaCount, hpaResourceRequest.ReqMaxReplicas, hpaResourceRequest.ReqMinReplicas) - templateMap["replicaCount"] = reqReplicaCount - merged, err = json.Marshal(&templateMap) - if err != nil { - impl.logger.Errorw("marshaling failed for hpa check", "err", err) - return merged - } - } - } else { - impl.logger.Errorw("autoscaling is not enabled", "pipelineId", pipelineId) - } - - //check for custom chart support - if autoscalingEnabledPath, ok := templateMap[bean2.CustomAutoScalingEnabledPathKey]; ok { - if deploymentType == models.DEPLOYMENTTYPE_STOP { - merged, err = impl.setScalingValues(templateMap, bean2.CustomAutoScalingEnabledPathKey, merged, false) - if err != nil { - return merged - } - merged, err = impl.setScalingValues(templateMap, bean2.CustomAutoscalingReplicaCountPathKey, merged, 0) - if err != nil { - return merged - } - } else { - autoscalingEnabled := false - autoscalingEnabledValue := gjson.Get(string(merged), autoscalingEnabledPath.(string)).Value() - if val, ok := autoscalingEnabledValue.(bool); ok { - autoscalingEnabled = val - } - if autoscalingEnabled { - // extract replica count, min, max and check for required value - replicaCount, err := impl.getReplicaCountFromCustomChart(templateMap, merged) - if err != nil { - return merged - } - merged, err = impl.setScalingValues(templateMap, bean2.CustomAutoscalingReplicaCountPathKey, merged, replicaCount) - if err != nil { - return merged - } - } - } - } - - return merged -} - func (impl *WorkflowDagExecutorImpl) updatePipeline(pipeline *pipelineConfig.Pipeline, userId int32) (bool, error) { err := impl.pipelineRepository.SetDeploymentAppCreatedInPipeline(true, pipeline.Id, userId) if err != nil { @@ -3530,174 +2637,6 @@ func (impl *WorkflowDagExecutorImpl) updatePipeline(pipeline *pipelineConfig.Pip return true, nil } -func (impl *WorkflowDagExecutorImpl) checkAndFixDuplicateReleaseNo(override *chartConfig.PipelineOverride) error { - - uniqueVerified := false - retryCount := 0 - - for !uniqueVerified && retryCount < 5 { - retryCount = retryCount + 1 - overrides, err := impl.pipelineOverrideRepository.GetByPipelineIdAndReleaseNo(override.PipelineId, override.PipelineReleaseCounter) - if err != nil { - return err - } - if overrides[0].Id == override.Id { - uniqueVerified = true - } else { - //duplicate might be due to concurrency, lets fix it - currentReleaseNo, err := impl.pipelineOverrideRepository.GetCurrentPipelineReleaseCounter(override.PipelineId) - if err != nil { - return err - } - override.PipelineReleaseCounter = currentReleaseNo + 1 - err = impl.pipelineOverrideRepository.Save(override) - if err != nil { - return err - } - } - } - if !uniqueVerified { - return fmt.Errorf("duplicate verification retry count exide max overrideId: %d ,count: %d", override.Id, retryCount) - } - return nil -} - -func (impl *WorkflowDagExecutorImpl) getAutoScalingReplicaCount(templateMap map[string]interface{}, appName string) *util4.HpaResourceRequest { - hasOverride := false - if _, ok := templateMap[fullnameOverride]; ok { - appNameOverride := templateMap[fullnameOverride].(string) - if len(appNameOverride) > 0 { - appName = appNameOverride - hasOverride = true - } - } - if !hasOverride { - if _, ok := templateMap[nameOverride]; ok { - nameOverride := templateMap[nameOverride].(string) - if len(nameOverride) > 0 { - appName = fmt.Sprintf("%s-%s", appName, nameOverride) - } - } - } - hpaResourceRequest := &util4.HpaResourceRequest{} - hpaResourceRequest.Version = "" - hpaResourceRequest.Group = autoscaling.ServiceName - hpaResourceRequest.Kind = horizontalPodAutoscaler - impl.logger.Infow("getAutoScalingReplicaCount", "hpaResourceRequest", hpaResourceRequest) - if _, ok := templateMap[kedaAutoscaling]; ok { - as := templateMap[kedaAutoscaling] - asd := as.(map[string]interface{}) - if _, ok := asd[enabled]; ok { - impl.logger.Infow("getAutoScalingReplicaCount", "hpaResourceRequest", hpaResourceRequest) - enable := asd[enabled].(bool) - if enable { - hpaResourceRequest.IsEnable = enable - hpaResourceRequest.ReqReplicaCount = templateMap[replicaCount].(float64) - hpaResourceRequest.ReqMaxReplicas = asd["maxReplicaCount"].(float64) - hpaResourceRequest.ReqMinReplicas = asd["minReplicaCount"].(float64) - hpaResourceRequest.ResourceName = fmt.Sprintf("%s-%s-%s", "keda-hpa", appName, "keda") - impl.logger.Infow("getAutoScalingReplicaCount", "hpaResourceRequest", hpaResourceRequest) - return hpaResourceRequest - } - } - } - - if _, ok := templateMap[autoscaling.ServiceName]; ok { - as := templateMap[autoscaling.ServiceName] - asd := as.(map[string]interface{}) - if _, ok := asd[enabled]; ok { - enable := asd[enabled].(bool) - if enable { - hpaResourceRequest.IsEnable = asd[enabled].(bool) - hpaResourceRequest.ReqReplicaCount = templateMap[replicaCount].(float64) - hpaResourceRequest.ReqMaxReplicas = asd["MaxReplicas"].(float64) - hpaResourceRequest.ReqMinReplicas = asd["MinReplicas"].(float64) - hpaResourceRequest.ResourceName = fmt.Sprintf("%s-%s", appName, "hpa") - return hpaResourceRequest - } - } - } - return hpaResourceRequest - -} - -func (impl *WorkflowDagExecutorImpl) fetchRequiredReplicaCount(currentReplicaCount float64, reqMaxReplicas float64, reqMinReplicas float64) float64 { - var reqReplicaCount float64 - if currentReplicaCount <= reqMaxReplicas && currentReplicaCount >= reqMinReplicas { - reqReplicaCount = currentReplicaCount - } else if currentReplicaCount > reqMaxReplicas { - reqReplicaCount = reqMaxReplicas - } else if currentReplicaCount < reqMinReplicas { - reqReplicaCount = reqMinReplicas - } - return reqReplicaCount -} - -func (impl *WorkflowDagExecutorImpl) getReplicaCountFromCustomChart(templateMap map[string]interface{}, merged []byte) (float64, error) { - autoscalingMinVal, err := impl.extractParamValue(templateMap, bean2.CustomAutoscalingMinPathKey, merged) - if err != nil { - return 0, err - } - autoscalingMaxVal, err := impl.extractParamValue(templateMap, bean2.CustomAutoscalingMaxPathKey, merged) - if err != nil { - return 0, err - } - autoscalingReplicaCountVal, err := impl.extractParamValue(templateMap, bean2.CustomAutoscalingReplicaCountPathKey, merged) - if err != nil { - return 0, err - } - return impl.fetchRequiredReplicaCount(autoscalingReplicaCountVal, autoscalingMaxVal, autoscalingMinVal), nil -} - -func (impl *WorkflowDagExecutorImpl) setScalingValues(templateMap map[string]interface{}, customScalingKey string, merged []byte, value interface{}) ([]byte, error) { - autoscalingJsonPath := templateMap[customScalingKey] - autoscalingJsonPathKey := autoscalingJsonPath.(string) - mergedRes, err := sjson.Set(string(merged), autoscalingJsonPathKey, value) - if err != nil { - impl.logger.Errorw("error occurred while setting autoscaling key", "JsonPathKey", autoscalingJsonPathKey, "err", err) - return []byte{}, err - } - return []byte(mergedRes), nil -} - -func (impl *WorkflowDagExecutorImpl) extractParamValue(inputMap map[string]interface{}, key string, merged []byte) (float64, error) { - if _, ok := inputMap[key]; !ok { - return 0, errors.New("empty-val-err") - } - floatNumber, err := util4.ParseFloatNumber(gjson.Get(string(merged), inputMap[key].(string)).Value()) - if err != nil { - impl.logger.Errorw("error occurred while parsing float number", "key", key, "err", err) - } - return floatNumber, err -} - -func (impl *WorkflowDagExecutorImpl) UpdatePreviousQueuedRunnerStatus(cdWfrId, pipelineId int, triggeredBy int32) error { - cdWfrs, err := impl.cdWorkflowRepository.UpdatePreviousQueuedRunnerStatus(cdWfrId, pipelineId, triggeredBy) - if err != nil { - impl.logger.Errorw("error on update previous queued cd workflow runner, UpdatePreviousQueuedRunnerStatus", "cdWfrId", cdWfrId, "err", err) - return err - } - for _, cdWfr := range cdWfrs { - err = impl.pipelineStatusTimelineService.MarkPipelineStatusTimelineFailed(cdWfr.Id, pipelineConfig.NEW_DEPLOYMENT_INITIATED) - if err != nil { - impl.logger.Errorw("error updating CdPipelineStatusTimeline, UpdatePreviousQueuedRunnerStatus", "err", err) - return err - } - if cdWfr.CdWorkflow == nil { - pipeline, err := impl.pipelineRepository.FindById(pipelineId) - if err != nil { - impl.logger.Errorw("error in fetching cd pipeline, UpdatePreviousQueuedRunnerStatus", "pipelineId", pipelineId, "err", err) - return err - } - cdWfr.CdWorkflow = &pipelineConfig.CdWorkflow{ - Pipeline: pipeline, - } - } - util4.TriggerCDMetrics(pipelineConfig.GetTriggerMetricsFromRunnerObj(cdWfr), impl.config.ExposeCDMetrics) - } - return nil -} - // canInitiateTrigger checks if the current trigger request with natsMsgId haven't already initiated the trigger. // throws error if the request is already processed. func (impl *WorkflowDagExecutorImpl) canInitiateTrigger(natsMsgId string) (bool, error) { diff --git a/pkg/pipeline/history/DeployedConfigurationHistoryService.go b/pkg/pipeline/history/DeployedConfigurationHistoryService.go index 788fba4cd2..4a582f1e3d 100644 --- a/pkg/pipeline/history/DeployedConfigurationHistoryService.go +++ b/pkg/pipeline/history/DeployedConfigurationHistoryService.go @@ -4,6 +4,11 @@ import ( "context" "errors" "fmt" + "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" + "github.com/devtron-labs/devtron/pkg/variables" + repository5 "github.com/devtron-labs/devtron/pkg/variables/repository" + util4 "github.com/devtron-labs/devtron/util" + "time" "github.com/devtron-labs/devtron/api/bean" repository2 "github.com/devtron-labs/devtron/internal/sql/repository" @@ -15,6 +20,9 @@ import ( ) type DeployedConfigurationHistoryService interface { + //TODO: rethink if the below method right at this place + CreateHistoriesForDeploymentTrigger(pipeline *pipelineConfig.Pipeline, strategy *chartConfig.PipelineStrategy, envOverride *chartConfig.EnvConfigOverride, deployedOn time.Time, deployedBy int32) error + GetDeployedConfigurationByWfrId(pipelineId, wfrId int) ([]*DeploymentConfigurationDto, error) GetDeployedHistoryComponentList(pipelineId, baseConfigId int, historyComponent, historyComponentName string) ([]*DeployedHistoryComponentMetadataDto, error) GetDeployedHistoryComponentDetail(ctx context.Context, pipelineId, id int, historyComponent, historyComponentName string, userHasAdminAccess bool) (*HistoryDetailDto, error) @@ -30,12 +38,14 @@ type DeployedConfigurationHistoryServiceImpl struct { strategyHistoryService PipelineStrategyHistoryService configMapHistoryService ConfigMapHistoryService cdWorkflowRepository pipelineConfig.CdWorkflowRepository + scopedVariableManager variables.ScopedVariableCMCSManager } func NewDeployedConfigurationHistoryServiceImpl(logger *zap.SugaredLogger, userService user.UserService, deploymentTemplateHistoryService DeploymentTemplateHistoryService, strategyHistoryService PipelineStrategyHistoryService, configMapHistoryService ConfigMapHistoryService, - cdWorkflowRepository pipelineConfig.CdWorkflowRepository) *DeployedConfigurationHistoryServiceImpl { + cdWorkflowRepository pipelineConfig.CdWorkflowRepository, + scopedVariableManager variables.ScopedVariableCMCSManager) *DeployedConfigurationHistoryServiceImpl { return &DeployedConfigurationHistoryServiceImpl{ logger: logger, userService: userService, @@ -43,7 +53,42 @@ func NewDeployedConfigurationHistoryServiceImpl(logger *zap.SugaredLogger, strategyHistoryService: strategyHistoryService, configMapHistoryService: configMapHistoryService, cdWorkflowRepository: cdWorkflowRepository, + scopedVariableManager: scopedVariableManager, + } +} + +func (impl *DeployedConfigurationHistoryServiceImpl) CreateHistoriesForDeploymentTrigger(pipeline *pipelineConfig.Pipeline, strategy *chartConfig.PipelineStrategy, envOverride *chartConfig.EnvConfigOverride, deployedOn time.Time, deployedBy int32) error { + //creating history for deployment template + deploymentTemplateHistory, err := impl.deploymentTemplateHistoryService.CreateDeploymentTemplateHistoryForDeploymentTrigger(pipeline, envOverride, envOverride.Chart.ImageDescriptorTemplate, deployedOn, deployedBy) + if err != nil { + impl.logger.Errorw("error in creating deployment template history for deployment trigger", "err", err) + return err + } + cmId, csId, err := impl.configMapHistoryService.CreateCMCSHistoryForDeploymentTrigger(pipeline, deployedOn, deployedBy) + if err != nil { + impl.logger.Errorw("error in creating CM/CS history for deployment trigger", "err", err) + return err + } + if strategy != nil { + err = impl.strategyHistoryService.CreateStrategyHistoryForDeploymentTrigger(strategy, deployedOn, deployedBy, pipeline.TriggerType) + if err != nil { + impl.logger.Errorw("error in creating strategy history for deployment trigger", "err", err) + return err + } + } + + var variableSnapshotHistories = util4.GetBeansPtr( + repository5.GetSnapshotBean(deploymentTemplateHistory.Id, repository5.HistoryReferenceTypeDeploymentTemplate, envOverride.VariableSnapshot), + repository5.GetSnapshotBean(cmId, repository5.HistoryReferenceTypeConfigMap, envOverride.VariableSnapshotForCM), + repository5.GetSnapshotBean(csId, repository5.HistoryReferenceTypeSecret, envOverride.VariableSnapshotForCS), + ) + if len(variableSnapshotHistories) > 0 { + err = impl.scopedVariableManager.SaveVariableHistoriesForTrigger(variableSnapshotHistories, deployedBy) + if err != nil { + return err + } } + return nil } func (impl *DeployedConfigurationHistoryServiceImpl) GetLatestDeployedArtifactByPipelineId(pipelineId int) (*repository2.CiArtifact, error) { diff --git a/util/GlobalConfig.go b/util/GlobalConfig.go index faee25ef9e..1c46f2f25d 100644 --- a/util/GlobalConfig.go +++ b/util/GlobalConfig.go @@ -7,8 +7,9 @@ import ( ) type GlobalEnvVariables struct { - GitOpsRepoPrefix string `env:"GITOPS_REPO_PREFIX" envDefault:""` - SkipGitOpsValidation bool `env:"SKIP_GITOPS_VALIDATION" envDefault:"false"` + GitOpsRepoPrefix string `env:"GITOPS_REPO_PREFIX" envDefault:""` + SkipGitOpsValidation bool `env:"SKIP_GITOPS_VALIDATION" envDefault:"false"` + EnableAsyncInstallDevtronChart bool `env:"ENABLE_ASYNC_INSTALL_DEVTRON_CHART" envDefault:"false"` } func GetGlobalEnvVariables() (*GlobalEnvVariables, error) { diff --git a/wire_gen.go b/wire_gen.go index 2e49d4115a..493989bccd 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -49,7 +49,7 @@ import ( "github.com/devtron-labs/devtron/client/argocdServer/application" "github.com/devtron-labs/devtron/client/argocdServer/cluster" "github.com/devtron-labs/devtron/client/argocdServer/connection" - repository13 "github.com/devtron-labs/devtron/client/argocdServer/repository" + repository11 "github.com/devtron-labs/devtron/client/argocdServer/repository" cron2 "github.com/devtron-labs/devtron/client/cron" "github.com/devtron-labs/devtron/client/dashboard" "github.com/devtron-labs/devtron/client/events" @@ -107,8 +107,9 @@ import ( delete2 "github.com/devtron-labs/devtron/pkg/delete" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" + "github.com/devtron-labs/devtron/pkg/deployment/manifest" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" - repository11 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/repository" + repository12 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/repository" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps" @@ -122,7 +123,7 @@ import ( "github.com/devtron-labs/devtron/pkg/externalLink" "github.com/devtron-labs/devtron/pkg/generateManifest" "github.com/devtron-labs/devtron/pkg/genericNotes" - repository12 "github.com/devtron-labs/devtron/pkg/genericNotes/repository" + repository13 "github.com/devtron-labs/devtron/pkg/genericNotes/repository" git2 "github.com/devtron-labs/devtron/pkg/git" "github.com/devtron-labs/devtron/pkg/gitops" "github.com/devtron-labs/devtron/pkg/imageDigestPolicy" @@ -387,7 +388,8 @@ func InitializeApp() (*App, error) { return nil, err } gitOperationServiceImpl := git.NewGitOperationServiceImpl(sugaredLogger, gitFactory, gitOpsConfigReadServiceImpl, chartTemplateServiceImpl) - appServiceImpl := app2.NewAppService(envConfigOverrideRepositoryImpl, pipelineOverrideRepositoryImpl, mergeUtil, sugaredLogger, pipelineRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, applicationServiceClientImpl, appRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, chartTemplateServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, appStatusServiceImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, globalEnvVariables, scopedVariableCMCSManagerImpl, acdConfig, chartRefServiceImpl, gitOpsConfigReadServiceImpl, gitOperationServiceImpl) + deploymentTemplateServiceImpl := deploymentTemplate.NewDeploymentTemplateServiceImpl(sugaredLogger, chartRefServiceImpl, chartTemplateServiceImpl, chartRepositoryImpl) + appServiceImpl := app2.NewAppService(envConfigOverrideRepositoryImpl, pipelineOverrideRepositoryImpl, mergeUtil, sugaredLogger, pipelineRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, applicationServiceClientImpl, appRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, chartTemplateServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, appStatusServiceImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, globalEnvVariables, scopedVariableCMCSManagerImpl, acdConfig, chartRefServiceImpl, gitOpsConfigReadServiceImpl, gitOperationServiceImpl, deploymentTemplateServiceImpl) validate, err := util.IntValidator() if err != nil { return nil, err @@ -438,21 +440,6 @@ func InitializeApp() (*App, error) { imageTagRepositoryImpl := repository.NewImageTagRepository(db, sugaredLogger) customTagServiceImpl := pipeline.NewCustomTagService(sugaredLogger, imageTagRepositoryImpl) pluginInputVariableParserImpl := pipeline.NewPluginInputVariableParserImpl(sugaredLogger, dockerRegistryConfigImpl, customTagServiceImpl) - deploymentTemplateHistoryRepositoryImpl := repository8.NewDeploymentTemplateHistoryRepositoryImpl(sugaredLogger, db) - appLevelMetricsRepositoryImpl := repository11.NewAppLevelMetricsRepositoryImpl(db, sugaredLogger) - envLevelAppMetricsRepositoryImpl := repository11.NewEnvLevelAppMetricsRepositoryImpl(db, sugaredLogger) - deployedAppMetricsServiceImpl := deployedAppMetrics.NewDeployedAppMetricsServiceImpl(sugaredLogger, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, chartRefServiceImpl) - deploymentTemplateHistoryServiceImpl := history.NewDeploymentTemplateHistoryServiceImpl(sugaredLogger, deploymentTemplateHistoryRepositoryImpl, pipelineRepositoryImpl, chartRepositoryImpl, userServiceImpl, cdWorkflowRepositoryImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) - pipelineStrategyHistoryRepositoryImpl := repository8.NewPipelineStrategyHistoryRepositoryImpl(sugaredLogger, db) - pipelineStrategyHistoryServiceImpl := history.NewPipelineStrategyHistoryServiceImpl(sugaredLogger, pipelineStrategyHistoryRepositoryImpl, userServiceImpl) - genericNoteRepositoryImpl := repository12.NewGenericNoteRepositoryImpl(db) - genericNoteHistoryRepositoryImpl := repository12.NewGenericNoteHistoryRepositoryImpl(db) - genericNoteHistoryServiceImpl := genericNotes.NewGenericNoteHistoryServiceImpl(genericNoteHistoryRepositoryImpl, sugaredLogger) - genericNoteServiceImpl := genericNotes.NewGenericNoteServiceImpl(genericNoteRepositoryImpl, genericNoteHistoryServiceImpl, userRepositoryImpl, sugaredLogger) - appCrudOperationServiceImpl := app2.NewAppCrudOperationServiceImpl(appLabelRepositoryImpl, sugaredLogger, appRepositoryImpl, userRepositoryImpl, installedAppRepositoryImpl, genericNoteServiceImpl, materialRepositoryImpl) - pipelineConfigRepositoryImpl := chartConfig.NewPipelineConfigRepository(db) - ciTemplateOverrideRepositoryImpl := pipelineConfig.NewCiTemplateOverrideRepositoryImpl(db, sugaredLogger) - dockerRegistryIpsConfigServiceImpl := dockerRegistry.NewDockerRegistryIpsConfigServiceImpl(sugaredLogger, dockerRegistryIpsConfigRepositoryImpl, k8sServiceImpl, clusterServiceImplExtended, ciPipelineRepositoryImpl, dockerArtifactStoreRepositoryImpl, ciTemplateOverrideRepositoryImpl) pipelineConfigListenerServiceImpl := pipeline.NewPipelineConfigListenerServiceImpl(sugaredLogger) imageDigestPolicyServiceImpl := imageDigestPolicy.NewImageDigestPolicyServiceImpl(sugaredLogger, qualifierMappingServiceImpl, devtronResourceSearchableKeyServiceImpl) cdWorkflowCommonServiceImpl, err := cd.NewCdWorkflowCommonServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl, pipelineStatusTimelineServiceImpl, pipelineRepositoryImpl) @@ -461,18 +448,35 @@ func InitializeApp() (*App, error) { } gitOpsManifestPushServiceImpl := app2.NewGitOpsManifestPushServiceImpl(sugaredLogger, pipelineStatusTimelineServiceImpl, pipelineStatusTimelineRepositoryImpl, acdConfig, chartRefServiceImpl, gitOpsConfigReadServiceImpl, gitOperationServiceImpl) argoK8sClientImpl := argocdServer.NewArgoK8sClientImpl(sugaredLogger) - repositoryServiceClientImpl := repository13.NewServiceClientImpl(sugaredLogger, argoCDConnectionManagerImpl) + repositoryServiceClientImpl := repository11.NewServiceClientImpl(sugaredLogger, argoCDConnectionManagerImpl) argoClientWrapperServiceImpl := argocdServer.NewArgoClientWrapperServiceImpl(sugaredLogger, applicationServiceClientImpl, acdConfig, repositoryServiceClientImpl) imageScanHistoryRepositoryImpl := security.NewImageScanHistoryRepositoryImpl(db, sugaredLogger) imageScanDeployInfoRepositoryImpl := security.NewImageScanDeployInfoRepositoryImpl(db, sugaredLogger) manifestPushConfigRepositoryImpl := repository9.NewManifestPushConfigRepository(sugaredLogger, db) - triggerServiceImpl := devtronApps.NewTriggerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, argoK8sClientImpl, acdConfig, applicationServiceClientImpl, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, appServiceImpl, helmAppServiceImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl) - workflowDagExecutorImpl := pipeline.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, pubSubClientServiceImpl, appServiceImpl, workflowServiceImpl, ciArtifactRepositoryImpl, ciPipelineRepositoryImpl, materialRepositoryImpl, pipelineOverrideRepositoryImpl, userServiceImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, enforcerUtilImpl, eventSimpleFactoryImpl, eventRESTClientImpl, cvePolicyRepositoryImpl, imageScanResultRepositoryImpl, appWorkflowRepositoryImpl, prePostCdScriptHistoryServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineServiceImpl, ciTemplateRepositoryImpl, ciWorkflowRepositoryImpl, appLabelRepositoryImpl, clientImpl, pipelineStageServiceImpl, k8sCommonServiceImpl, globalPluginServiceImpl, pluginInputVariableParserImpl, scopedVariableCMCSManagerImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineStrategyHistoryServiceImpl, appCrudOperationServiceImpl, pipelineConfigRepositoryImpl, dockerRegistryIpsConfigServiceImpl, chartRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, configMapRepositoryImpl, configMapHistoryRepositoryImpl, helmAppServiceImpl, envConfigOverrideRepositoryImpl, mergeUtil, applicationServiceClientImpl, pipelineConfigListenerServiceImpl, customTagServiceImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, imageDigestPolicyServiceImpl, cdWorkflowCommonServiceImpl, triggerServiceImpl) - deploymentGroupAppRepositoryImpl := repository.NewDeploymentGroupAppRepositoryImpl(sugaredLogger, db) - workflowEventPublishServiceImpl, err := out.NewWorkflowEventPublishServiceImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowRepositoryImpl) + triggerServiceImpl := devtronApps.NewTriggerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, argoK8sClientImpl, acdConfig, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, helmAppServiceImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, globalEnvVariables, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl) + deploymentTemplateHistoryRepositoryImpl := repository8.NewDeploymentTemplateHistoryRepositoryImpl(sugaredLogger, db) + appLevelMetricsRepositoryImpl := repository12.NewAppLevelMetricsRepositoryImpl(db, sugaredLogger) + envLevelAppMetricsRepositoryImpl := repository12.NewEnvLevelAppMetricsRepositoryImpl(db, sugaredLogger) + deployedAppMetricsServiceImpl := deployedAppMetrics.NewDeployedAppMetricsServiceImpl(sugaredLogger, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, chartRefServiceImpl) + deploymentTemplateHistoryServiceImpl := history.NewDeploymentTemplateHistoryServiceImpl(sugaredLogger, deploymentTemplateHistoryRepositoryImpl, pipelineRepositoryImpl, chartRepositoryImpl, userServiceImpl, cdWorkflowRepositoryImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) + pipelineStrategyHistoryRepositoryImpl := repository8.NewPipelineStrategyHistoryRepositoryImpl(sugaredLogger, db) + pipelineStrategyHistoryServiceImpl := history.NewPipelineStrategyHistoryServiceImpl(sugaredLogger, pipelineStrategyHistoryRepositoryImpl, userServiceImpl) + deployedConfigurationHistoryServiceImpl := history.NewDeployedConfigurationHistoryServiceImpl(sugaredLogger, userServiceImpl, deploymentTemplateHistoryServiceImpl, pipelineStrategyHistoryServiceImpl, configMapHistoryServiceImpl, cdWorkflowRepositoryImpl, scopedVariableCMCSManagerImpl) + ciTemplateOverrideRepositoryImpl := pipelineConfig.NewCiTemplateOverrideRepositoryImpl(db, sugaredLogger) + dockerRegistryIpsConfigServiceImpl := dockerRegistry.NewDockerRegistryIpsConfigServiceImpl(sugaredLogger, dockerRegistryIpsConfigRepositoryImpl, k8sServiceImpl, clusterServiceImplExtended, ciPipelineRepositoryImpl, dockerArtifactStoreRepositoryImpl, ciTemplateOverrideRepositoryImpl) + genericNoteRepositoryImpl := repository13.NewGenericNoteRepositoryImpl(db) + genericNoteHistoryRepositoryImpl := repository13.NewGenericNoteHistoryRepositoryImpl(db) + genericNoteHistoryServiceImpl := genericNotes.NewGenericNoteHistoryServiceImpl(genericNoteHistoryRepositoryImpl, sugaredLogger) + genericNoteServiceImpl := genericNotes.NewGenericNoteServiceImpl(genericNoteRepositoryImpl, genericNoteHistoryServiceImpl, userRepositoryImpl, sugaredLogger) + appCrudOperationServiceImpl := app2.NewAppCrudOperationServiceImpl(appLabelRepositoryImpl, sugaredLogger, appRepositoryImpl, userRepositoryImpl, installedAppRepositoryImpl, genericNoteServiceImpl, materialRepositoryImpl) + pipelineConfigRepositoryImpl := chartConfig.NewPipelineConfigRepository(db) + manifestCreationServiceImpl := manifest.NewManifestCreationServiceImpl(sugaredLogger, dockerRegistryIpsConfigServiceImpl, chartRefServiceImpl, scopedVariableCMCSManagerImpl, k8sCommonServiceImpl, deployedAppMetricsServiceImpl, imageDigestPolicyServiceImpl, mergeUtil, appCrudOperationServiceImpl, applicationServiceClientImpl, configMapHistoryRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, envConfigOverrideRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineOverrideRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, pipelineConfigRepositoryImpl, deploymentTemplateHistoryRepositoryImpl) + workflowEventPublishServiceImpl, err := out.NewWorkflowEventPublishServiceImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowCommonServiceImpl, deployedConfigurationHistoryServiceImpl, manifestCreationServiceImpl, pipelineStatusTimelineServiceImpl, cdWorkflowRepositoryImpl, pipelineRepositoryImpl) if err != nil { return nil, err } + workflowDagExecutorImpl := pipeline.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, pubSubClientServiceImpl, workflowServiceImpl, ciArtifactRepositoryImpl, ciPipelineRepositoryImpl, materialRepositoryImpl, pipelineOverrideRepositoryImpl, userServiceImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, enforcerUtilImpl, eventSimpleFactoryImpl, eventRESTClientImpl, cvePolicyRepositoryImpl, imageScanResultRepositoryImpl, appWorkflowRepositoryImpl, prePostCdScriptHistoryServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineServiceImpl, ciTemplateRepositoryImpl, ciWorkflowRepositoryImpl, appLabelRepositoryImpl, clientImpl, pipelineStageServiceImpl, k8sCommonServiceImpl, globalPluginServiceImpl, pluginInputVariableParserImpl, scopedVariableCMCSManagerImpl, helmAppServiceImpl, pipelineConfigListenerServiceImpl, customTagServiceImpl, imageDigestPolicyServiceImpl, cdWorkflowCommonServiceImpl, triggerServiceImpl, deployedConfigurationHistoryServiceImpl, workflowEventPublishServiceImpl, manifestCreationServiceImpl, deploymentTemplateServiceImpl) + deploymentGroupAppRepositoryImpl := repository.NewDeploymentGroupAppRepositoryImpl(sugaredLogger, db) deploymentGroupServiceImpl := deploymentGroup.NewDeploymentGroupServiceImpl(appRepositoryImpl, sugaredLogger, pipelineRepositoryImpl, ciPipelineRepositoryImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, deploymentGroupAppRepositoryImpl, ciArtifactRepositoryImpl, appWorkflowRepositoryImpl, workflowEventPublishServiceImpl) deploymentConfigServiceImpl := pipeline.NewDeploymentConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, pipelineRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, configMapHistoryServiceImpl, scopedVariableCMCSManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) pipelineTriggerRestHandlerImpl := restHandler.NewPipelineRestHandler(appServiceImpl, userServiceImpl, validate, enforcerImpl, teamServiceImpl, sugaredLogger, enforcerUtilImpl, workflowDagExecutorImpl, deploymentGroupServiceImpl, argoUserServiceImpl, deploymentConfigServiceImpl) @@ -502,7 +506,7 @@ func InitializeApp() (*App, error) { resourceGroupMappingRepositoryImpl := resourceGroup.NewResourceGroupMappingRepositoryImpl(db) resourceGroupServiceImpl := resourceGroup2.NewResourceGroupServiceImpl(sugaredLogger, resourceGroupRepositoryImpl, resourceGroupMappingRepositoryImpl, enforcerUtilImpl, devtronResourceSearchableKeyServiceImpl, appStatusRepositoryImpl) buildPipelineSwitchServiceImpl := pipeline.NewBuildPipelineSwitchServiceImpl(sugaredLogger, ciPipelineRepositoryImpl, ciCdPipelineOrchestratorImpl, pipelineRepositoryImpl, ciWorkflowRepositoryImpl, appWorkflowRepositoryImpl, ciPipelineHistoryServiceImpl, ciTemplateOverrideRepositoryImpl, ciPipelineMaterialRepositoryImpl) - ciPipelineConfigServiceImpl := pipeline.NewCiPipelineConfigServiceImpl(sugaredLogger, ciCdPipelineOrchestratorImpl, dockerArtifactStoreRepositoryImpl, materialRepositoryImpl, appRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ecrConfig, appWorkflowRepositoryImpl, ciCdConfig, attributesServiceImpl, pipelineStageServiceImpl, ciPipelineMaterialRepositoryImpl, ciTemplateServiceImpl, ciTemplateOverrideRepositoryImpl, ciTemplateHistoryServiceImpl, enforcerUtilImpl, ciWorkflowRepositoryImpl, resourceGroupServiceImpl, customTagServiceImpl, ciPipelineHistoryServiceImpl, cdWorkflowRepositoryImpl, buildPipelineSwitchServiceImpl) + ciPipelineConfigServiceImpl := pipeline.NewCiPipelineConfigServiceImpl(sugaredLogger, ciCdPipelineOrchestratorImpl, dockerArtifactStoreRepositoryImpl, materialRepositoryImpl, appRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ecrConfig, appWorkflowRepositoryImpl, ciCdConfig, attributesServiceImpl, pipelineStageServiceImpl, ciPipelineMaterialRepositoryImpl, ciTemplateServiceImpl, ciTemplateOverrideRepositoryImpl, ciTemplateHistoryServiceImpl, enforcerUtilImpl, ciWorkflowRepositoryImpl, resourceGroupServiceImpl, customTagServiceImpl, cdWorkflowRepositoryImpl, buildPipelineSwitchServiceImpl) ciMaterialConfigServiceImpl := pipeline.NewCiMaterialConfigServiceImpl(sugaredLogger, materialRepositoryImpl, ciTemplateServiceImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, gitMaterialHistoryServiceImpl, pipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl) imageTaggingRepositoryImpl := repository14.NewImageTaggingRepositoryImpl(db) imageTaggingServiceImpl := pipeline.NewImageTaggingServiceImpl(imageTaggingRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, sugaredLogger) @@ -537,16 +541,15 @@ func InitializeApp() (*App, error) { appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, appRepositoryImpl, userAuthServiceImpl) appCloneServiceImpl := appClone.NewAppCloneServiceImpl(sugaredLogger, pipelineBuilderImpl, chartServiceImpl, configMapServiceImpl, appWorkflowServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, pipelineStageServiceImpl, ciTemplateServiceImpl, appRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, ciPipelineConfigServiceImpl, gitOpsConfigReadServiceImpl) deploymentTemplateRepositoryImpl := repository.NewDeploymentTemplateRepositoryImpl(db, sugaredLogger) - deploymentTemplateServiceImpl := generateManifest.NewDeploymentTemplateServiceImpl(sugaredLogger, chartServiceImpl, appListingServiceImpl, appListingRepositoryImpl, deploymentTemplateRepositoryImpl, helmAppServiceImpl, chartRepositoryImpl, chartTemplateServiceImpl, helmAppClientImpl, k8sServiceImpl, propertiesConfigServiceImpl, deploymentTemplateHistoryServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, chartRefServiceImpl) + generateManifestDeploymentTemplateServiceImpl := generateManifest.NewDeploymentTemplateServiceImpl(sugaredLogger, chartServiceImpl, appListingServiceImpl, appListingRepositoryImpl, deploymentTemplateRepositoryImpl, helmAppServiceImpl, chartRepositoryImpl, chartTemplateServiceImpl, helmAppClientImpl, k8sServiceImpl, propertiesConfigServiceImpl, deploymentTemplateHistoryServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, chartRefServiceImpl) imageScanObjectMetaRepositoryImpl := security.NewImageScanObjectMetaRepositoryImpl(db, sugaredLogger) cveStoreRepositoryImpl := security.NewCveStoreRepositoryImpl(db, sugaredLogger) policyServiceImpl := security2.NewPolicyServiceImpl(environmentServiceImpl, sugaredLogger, appRepositoryImpl, pipelineOverrideRepositoryImpl, cvePolicyRepositoryImpl, clusterServiceImplExtended, pipelineRepositoryImpl, imageScanResultRepositoryImpl, imageScanDeployInfoRepositoryImpl, imageScanObjectMetaRepositoryImpl, httpClient, ciArtifactRepositoryImpl, ciCdConfig, imageScanHistoryRepositoryImpl, cveStoreRepositoryImpl, ciTemplateRepositoryImpl) - pipelineConfigRestHandlerImpl := app3.NewPipelineRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, deploymentTemplateValidationServiceImpl, chartServiceImpl, propertiesConfigServiceImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, environmentServiceImpl, gitRegistryConfigImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, deploymentTemplateServiceImpl, appWorkflowServiceImpl, materialRepositoryImpl, policyServiceImpl, imageScanResultRepositoryImpl, gitProviderRepositoryImpl, argoUserServiceImpl, ciPipelineMaterialRepositoryImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) + pipelineConfigRestHandlerImpl := app3.NewPipelineRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, deploymentTemplateValidationServiceImpl, chartServiceImpl, propertiesConfigServiceImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, environmentServiceImpl, gitRegistryConfigImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, generateManifestDeploymentTemplateServiceImpl, appWorkflowServiceImpl, materialRepositoryImpl, policyServiceImpl, imageScanResultRepositoryImpl, gitProviderRepositoryImpl, argoUserServiceImpl, ciPipelineMaterialRepositoryImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) appWorkflowRestHandlerImpl := restHandler.NewAppWorkflowRestHandlerImpl(sugaredLogger, userServiceImpl, appWorkflowServiceImpl, teamServiceImpl, enforcerImpl, pipelineBuilderImpl, appRepositoryImpl, enforcerUtilImpl) webhookEventDataRepositoryImpl := repository.NewWebhookEventDataRepositoryImpl(db) webhookEventDataConfigImpl := pipeline.NewWebhookEventDataConfigImpl(sugaredLogger, webhookEventDataRepositoryImpl) webhookDataRestHandlerImpl := restHandler.NewWebhookDataRestHandlerImpl(sugaredLogger, userServiceImpl, ciPipelineMaterialRepositoryImpl, enforcerUtilImpl, enforcerImpl, clientImpl, webhookEventDataConfigImpl) - deployedConfigurationHistoryServiceImpl := history.NewDeployedConfigurationHistoryServiceImpl(sugaredLogger, userServiceImpl, deploymentTemplateHistoryServiceImpl, pipelineStrategyHistoryServiceImpl, configMapHistoryServiceImpl, cdWorkflowRepositoryImpl) pipelineHistoryRestHandlerImpl := restHandler.NewPipelineHistoryRestHandlerImpl(sugaredLogger, userServiceImpl, enforcerImpl, pipelineStrategyHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, prePostCiScriptHistoryServiceImpl, prePostCdScriptHistoryServiceImpl, enforcerUtilImpl, deployedConfigurationHistoryServiceImpl) pipelineStatusTimelineRestHandlerImpl := restHandler.NewPipelineStatusTimelineRestHandlerImpl(sugaredLogger, pipelineStatusTimelineServiceImpl, enforcerUtilImpl, enforcerImpl) pipelineConfigRouterImpl := router.NewPipelineRouterImpl(pipelineConfigRestHandlerImpl, appWorkflowRestHandlerImpl, webhookDataRestHandlerImpl, pipelineHistoryRestHandlerImpl, pipelineStatusTimelineRestHandlerImpl) @@ -580,7 +583,7 @@ func InitializeApp() (*App, error) { return nil, err } cdApplicationStatusUpdateHandlerImpl := cron2.NewCdApplicationStatusUpdateHandlerImpl(sugaredLogger, appServiceImpl, workflowDagExecutorImpl, installedAppServiceImpl, cdHandlerImpl, appServiceConfig, pubSubClientServiceImpl, pipelineStatusTimelineRepositoryImpl, eventRESTClientImpl, appListingRepositoryImpl, cdWorkflowRepositoryImpl, pipelineRepositoryImpl, installedAppVersionHistoryRepositoryImpl, installedAppRepositoryImpl, cronLoggerImpl) - appListingRestHandlerImpl := restHandler.NewAppListingRestHandlerImpl(applicationServiceClientImpl, appListingServiceImpl, teamServiceImpl, enforcerImpl, pipelineBuilderImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, userServiceImpl, helmAppClientImpl, clusterServiceImplExtended, helmAppServiceImpl, argoUserServiceImpl, k8sCommonServiceImpl, installedAppServiceImpl, cdApplicationStatusUpdateHandlerImpl, pipelineRepositoryImpl, appStatusServiceImpl, installedAppRepositoryImpl, environmentServiceImpl, genericNoteServiceImpl, k8sApplicationServiceImpl, deploymentTemplateServiceImpl) + appListingRestHandlerImpl := restHandler.NewAppListingRestHandlerImpl(applicationServiceClientImpl, appListingServiceImpl, teamServiceImpl, enforcerImpl, pipelineBuilderImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, userServiceImpl, helmAppClientImpl, clusterServiceImplExtended, helmAppServiceImpl, argoUserServiceImpl, k8sCommonServiceImpl, installedAppServiceImpl, cdApplicationStatusUpdateHandlerImpl, pipelineRepositoryImpl, appStatusServiceImpl, installedAppRepositoryImpl, environmentServiceImpl, genericNoteServiceImpl, k8sApplicationServiceImpl, generateManifestDeploymentTemplateServiceImpl) appListingRouterImpl := router.NewAppListingRouterImpl(appListingRestHandlerImpl) chartRepositoryServiceImpl := chartRepo.NewChartRepositoryServiceImpl(sugaredLogger, chartRepoRepositoryImpl, k8sServiceImpl, clusterServiceImplExtended, acdAuthConfig, httpClient, serverEnvConfigServerEnvConfig) deleteServiceExtendedImpl := delete2.NewDeleteServiceExtendedImpl(sugaredLogger, teamServiceImpl, clusterServiceImplExtended, environmentServiceImpl, appRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, chartRepositoryServiceImpl, installedAppRepositoryImpl, dockerRegistryConfigImpl, dockerArtifactStoreRepositoryImpl) From 8e0819e95d376c08b11b56f781b34c9f83dbf28b Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Fri, 2 Feb 2024 13:40:46 +0530 Subject: [PATCH 53/64] moved WorkflowStatusUpdateHandler --- Wire.go | 3 - .../pubsub/WorkflowStatusUpdateHandler.go | 199 ------------------ api/router/router.go | 3 - .../CentralEventProcessorService.go | 14 +- .../in/WorkflowEventProcessorService.go | 159 +++++++++++++- wire_gen.go | 5 +- 6 files changed, 171 insertions(+), 212 deletions(-) delete mode 100644 api/router/pubsub/WorkflowStatusUpdateHandler.go diff --git a/Wire.go b/Wire.go index d004fe107d..3366f1ca8d 100644 --- a/Wire.go +++ b/Wire.go @@ -451,9 +451,6 @@ func InitializeApp() (*App, error) { pubsub.NewGitWebhookHandler, wire.Bind(new(pubsub.GitWebhookHandler), new(*pubsub.GitWebhookHandlerImpl)), - pubsub.NewWorkflowStatusUpdateHandlerImpl, - wire.Bind(new(pubsub.WorkflowStatusUpdateHandler), new(*pubsub.WorkflowStatusUpdateHandlerImpl)), - pubsub.NewApplicationStatusHandlerImpl, wire.Bind(new(pubsub.ApplicationStatusHandler), new(*pubsub.ApplicationStatusHandlerImpl)), diff --git a/api/router/pubsub/WorkflowStatusUpdateHandler.go b/api/router/pubsub/WorkflowStatusUpdateHandler.go deleted file mode 100644 index f44c2af4a3..0000000000 --- a/api/router/pubsub/WorkflowStatusUpdateHandler.go +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Copyright (c) 2020 Devtron Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package pubsub - -import ( - "encoding/json" - "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" - pubsub "github.com/devtron-labs/common-lib/pubsub-lib" - "github.com/devtron-labs/common-lib/pubsub-lib/model" - "github.com/devtron-labs/devtron/api/bean" - client "github.com/devtron-labs/devtron/client/events" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/devtron-labs/devtron/pkg/pipeline" - "github.com/devtron-labs/devtron/pkg/pipeline/executors" - util "github.com/devtron-labs/devtron/util/event" - "go.uber.org/zap" -) - -type WorkflowStatusUpdateHandler interface { - Subscribe() error -} - -type WorkflowStatusUpdateHandlerImpl struct { - logger *zap.SugaredLogger - pubsubClient *pubsub.PubSubClientServiceImpl - ciHandler pipeline.CiHandler - cdHandler pipeline.CdHandler - eventFactory client.EventFactory - eventClient client.EventClient - cdWorkflowRepository pipelineConfig.CdWorkflowRepository -} - -func NewWorkflowStatusUpdateHandlerImpl(logger *zap.SugaredLogger, pubsubClient *pubsub.PubSubClientServiceImpl, ciHandler pipeline.CiHandler, cdHandler pipeline.CdHandler, - eventFactory client.EventFactory, eventClient client.EventClient, cdWorkflowRepository pipelineConfig.CdWorkflowRepository) *WorkflowStatusUpdateHandlerImpl { - workflowStatusUpdateHandlerImpl := &WorkflowStatusUpdateHandlerImpl{ - logger: logger, - pubsubClient: pubsubClient, - ciHandler: ciHandler, - cdHandler: cdHandler, - eventFactory: eventFactory, - eventClient: eventClient, - cdWorkflowRepository: cdWorkflowRepository, - } - err := workflowStatusUpdateHandlerImpl.Subscribe() - if err != nil { - logger.Error("err", err) - return nil - } - err = workflowStatusUpdateHandlerImpl.SubscribeCD() - if err != nil { - logger.Error("err", err) - return nil - } - return workflowStatusUpdateHandlerImpl -} - -func (impl *WorkflowStatusUpdateHandlerImpl) Subscribe() error { - callback := func(msg *model.PubSubMsg) { - wfStatus := v1alpha1.WorkflowStatus{} - err := json.Unmarshal([]byte(string(msg.Data)), &wfStatus) - if err != nil { - impl.logger.Errorw("error while unmarshalling wf status update", "err", err, "msg", string(msg.Data)) - return - } - - err = impl.ciHandler.CheckAndReTriggerCI(wfStatus) - if err != nil { - impl.logger.Errorw("error in checking and re triggering ci", "err", err) - //don't return as we have to update the workflow status - } - - _, err = impl.ciHandler.UpdateWorkflow(wfStatus) - if err != nil { - impl.logger.Errorw("error on update workflow status", "err", err, "msg", string(msg.Data)) - return - } - - } - - // add required logging here - var loggerFunc pubsub.LoggerFunc = func(msg model.PubSubMsg) (string, []interface{}) { - wfStatus := v1alpha1.WorkflowStatus{} - err := json.Unmarshal([]byte(string(msg.Data)), &wfStatus) - if err != nil { - return "error while unmarshalling wf status update", []interface{}{"err", err, "msg", string(msg.Data)} - } - - workflowName, status, _, message, _, _ := pipeline.ExtractWorkflowStatus(wfStatus) - return "got message for ci workflow status update ", []interface{}{"workflowName", workflowName, "status", status, "message", message} - } - - err := impl.pubsubClient.Subscribe(pubsub.WORKFLOW_STATUS_UPDATE_TOPIC, callback, loggerFunc) - - if err != nil { - impl.logger.Error("err", err) - return err - } - return nil -} - -func (impl *WorkflowStatusUpdateHandlerImpl) SubscribeCD() error { - callback := func(msg *model.PubSubMsg) { - wfStatus := v1alpha1.WorkflowStatus{} - err := json.Unmarshal([]byte(string(msg.Data)), &wfStatus) - if err != nil { - impl.logger.Error("Error while unmarshalling wfStatus json object", "error", err) - return - } - - wfrId, wfrStatus, err := impl.cdHandler.UpdateWorkflow(wfStatus) - impl.logger.Debugw("UpdateWorkflow", "wfrId", wfrId, "wfrStatus", wfrStatus) - if err != nil { - impl.logger.Error("err", err) - return - } - - wfr, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(wfrId) - if err != nil { - impl.logger.Errorw("could not get wf runner", "err", err) - return - } - if wfrStatus == string(v1alpha1.NodeFailed) || wfrStatus == string(v1alpha1.NodeError) { - if len(wfr.ImagePathReservationIds) > 0 { - err := impl.cdHandler.DeactivateImageReservationPathsOnFailure(wfr.ImagePathReservationIds) - if err != nil { - impl.logger.Errorw("error in removing image path reservation ") - } - } - } - if wfrStatus == string(v1alpha1.NodeSucceeded) || wfrStatus == string(v1alpha1.NodeFailed) || wfrStatus == string(v1alpha1.NodeError) { - eventType := util.EventType(0) - if wfrStatus == string(v1alpha1.NodeSucceeded) { - eventType = util.Success - } else if wfrStatus == string(v1alpha1.NodeFailed) || wfrStatus == string(v1alpha1.NodeError) { - eventType = util.Fail - } - - if wfr != nil && executors.CheckIfReTriggerRequired(wfrStatus, wfStatus.Message, wfr.Status) { - err = impl.cdHandler.HandleCdStageReTrigger(wfr) - if err != nil { - //check if this log required or not - impl.logger.Errorw("error in HandleCdStageReTrigger", "error", err) - } - } - - if wfr.WorkflowType == bean.CD_WORKFLOW_TYPE_PRE { - event := impl.eventFactory.Build(eventType, &wfr.CdWorkflow.PipelineId, wfr.CdWorkflow.Pipeline.AppId, &wfr.CdWorkflow.Pipeline.EnvironmentId, util.CD) - impl.logger.Debugw("event pre stage", "event", event) - event = impl.eventFactory.BuildExtraCDData(event, wfr, 0, bean.CD_WORKFLOW_TYPE_PRE) - _, evtErr := impl.eventClient.WriteNotificationEvent(event) - if evtErr != nil { - impl.logger.Errorw("CD stage post fail or success event unable to sent", "error", evtErr) - } - - } else if wfr.WorkflowType == bean.CD_WORKFLOW_TYPE_POST { - event := impl.eventFactory.Build(eventType, &wfr.CdWorkflow.PipelineId, wfr.CdWorkflow.Pipeline.AppId, &wfr.CdWorkflow.Pipeline.EnvironmentId, util.CD) - impl.logger.Debugw("event post stage", "event", event) - event = impl.eventFactory.BuildExtraCDData(event, wfr, 0, bean.CD_WORKFLOW_TYPE_POST) - _, evtErr := impl.eventClient.WriteNotificationEvent(event) - if evtErr != nil { - impl.logger.Errorw("CD stage post fail or success event not sent", "error", evtErr) - } - } - } - } - - // add required logging here - var loggerFunc pubsub.LoggerFunc = func(msg model.PubSubMsg) (string, []interface{}) { - wfStatus := v1alpha1.WorkflowStatus{} - err := json.Unmarshal([]byte(string(msg.Data)), &wfStatus) - if err != nil { - return "error while unmarshalling wfStatus json object", []interface{}{"error", err} - } - workflowName, status, _, message, _, _ := pipeline.ExtractWorkflowStatus(wfStatus) - return "got message for cd workflow status", []interface{}{"workflowName", workflowName, "status", status, "message", message} - } - - err := impl.pubsubClient.Subscribe(pubsub.CD_WORKFLOW_STATUS_UPDATE, callback, loggerFunc) - if err != nil { - impl.logger.Error("err", err) - return err - } - return nil -} diff --git a/api/router/router.go b/api/router/router.go index 58375546ce..4c7f779bdf 100644 --- a/api/router/router.go +++ b/api/router/router.go @@ -74,7 +74,6 @@ type MuxRouter struct { pubsubClient *pubsub2.PubSubClientServiceImpl UserRouter user.UserRouter gitWebhookHandler pubsub.GitWebhookHandler - workflowUpdateHandler pubsub.WorkflowStatusUpdateHandler appUpdateHandler pubsub.ApplicationStatusHandler ciEventHandler pubsub.CiEventHandler ChartRefRouter ChartRefRouter @@ -134,7 +133,6 @@ func NewMuxRouter(logger *zap.SugaredLogger, HelmRouter PipelineTriggerRouter, P NotificationRouter NotificationRouter, TeamRouter team.TeamRouter, gitWebhookHandler pubsub.GitWebhookHandler, - workflowUpdateHandler pubsub.WorkflowStatusUpdateHandler, appUpdateHandler pubsub.ApplicationStatusHandler, ciEventHandler pubsub.CiEventHandler, pubsubClient *pubsub2.PubSubClientServiceImpl, UserRouter user.UserRouter, ChartRefRouter ChartRefRouter, ConfigMapRouter ConfigMapRouter, AppStoreRouter appStore.AppStoreRouter, chartRepositoryRouter chartRepo.ChartRepositoryRouter, @@ -173,7 +171,6 @@ func NewMuxRouter(logger *zap.SugaredLogger, HelmRouter PipelineTriggerRouter, P TeamRouter: TeamRouter, logger: logger, gitWebhookHandler: gitWebhookHandler, - workflowUpdateHandler: workflowUpdateHandler, appUpdateHandler: appUpdateHandler, ciEventHandler: ciEventHandler, pubsubClient: pubsubClient, diff --git a/pkg/eventProcessor/CentralEventProcessorService.go b/pkg/eventProcessor/CentralEventProcessorService.go index a7d36477c0..5fa258499c 100644 --- a/pkg/eventProcessor/CentralEventProcessorService.go +++ b/pkg/eventProcessor/CentralEventProcessorService.go @@ -25,9 +25,9 @@ func NewCentralEventProcessor(workflowEventProcessor *in.WorkflowEventProcessorI func (impl *CentralEventProcessor) SubscribeAll() error { var err error - err = impl.workflowEventProcessor.SubscribeCdStageCompleteEvent() + err = impl.workflowEventProcessor.SubscribeCDStageCompleteEvent() if err != nil { - impl.logger.Errorw("error, SubscribeCdStageCompleteEvent", "err", err) + impl.logger.Errorw("error, SubscribeCDStageCompleteEvent", "err", err) return err } err = impl.workflowEventProcessor.SubscribeTriggerBulkAction() @@ -40,5 +40,15 @@ func (impl *CentralEventProcessor) SubscribeAll() error { impl.logger.Errorw("error, SubscribeHibernateBulkAction", "err", err) return err } + err = impl.workflowEventProcessor.SubscribeCIWorkflowStatusUpdate() + if err != nil { + impl.logger.Errorw("error, SubscribeCIWorkflowStatusUpdate", "err", err) + return err + } + err = impl.workflowEventProcessor.SubscribeCDWorkflowStatusUpdate() + if err != nil { + impl.logger.Errorw("error, SubscribeCDWorkflowStatusUpdate", "err", err) + return err + } return nil } diff --git a/pkg/eventProcessor/in/WorkflowEventProcessorService.go b/pkg/eventProcessor/in/WorkflowEventProcessorService.go index ee1576c886..51cc7576ab 100644 --- a/pkg/eventProcessor/in/WorkflowEventProcessorService.go +++ b/pkg/eventProcessor/in/WorkflowEventProcessorService.go @@ -2,18 +2,22 @@ package in import ( "encoding/json" + "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" pubsub "github.com/devtron-labs/common-lib/pubsub-lib" "github.com/devtron-labs/common-lib/pubsub-lib/model" bean2 "github.com/devtron-labs/devtron/api/bean" + client "github.com/devtron-labs/devtron/client/events" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" bean4 "github.com/devtron-labs/devtron/pkg/auth/user/bean" "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/workflow/cd" "github.com/devtron-labs/devtron/pkg/workflow/cd/adapter" bean3 "github.com/devtron-labs/devtron/pkg/workflow/cd/bean" "github.com/devtron-labs/devtron/util/argo" + util "github.com/devtron-labs/devtron/util/event" "go.uber.org/zap" "k8s.io/utils/pointer" "strconv" @@ -26,9 +30,14 @@ type WorkflowEventProcessorImpl struct { cdWorkflowRunnerService cd.CdWorkflowRunnerService workflowDagExecutor pipeline.WorkflowDagExecutor argoUserService argo.ArgoUserService + ciHandler pipeline.CiHandler + cdHandler pipeline.CdHandler + eventFactory client.EventFactory + eventClient client.EventClient //repositories import, to be removed pipelineRepository pipelineConfig.PipelineRepository ciArtifactRepository repository.CiArtifactRepository + cdWorkflowRepository pipelineConfig.CdWorkflowRepository } func NewWorkflowEventProcessorImpl(logger *zap.SugaredLogger, @@ -37,22 +46,30 @@ func NewWorkflowEventProcessorImpl(logger *zap.SugaredLogger, cdWorkflowRunnerService cd.CdWorkflowRunnerService, workflowDagExecutor pipeline.WorkflowDagExecutor, argoUserService argo.ArgoUserService, + ciHandler pipeline.CiHandler, cdHandler pipeline.CdHandler, + eventFactory client.EventFactory, eventClient client.EventClient, pipelineRepository pipelineConfig.PipelineRepository, - ciArtifactRepository repository.CiArtifactRepository) (*WorkflowEventProcessorImpl, error) { + ciArtifactRepository repository.CiArtifactRepository, + cdWorkflowRepository pipelineConfig.CdWorkflowRepository) (*WorkflowEventProcessorImpl, error) { impl := &WorkflowEventProcessorImpl{ logger: logger, pubSubClient: pubSubClient, cdWorkflowService: cdWorkflowService, cdWorkflowRunnerService: cdWorkflowRunnerService, argoUserService: argoUserService, + ciHandler: ciHandler, + cdHandler: cdHandler, + eventFactory: eventFactory, + eventClient: eventClient, workflowDagExecutor: workflowDagExecutor, pipelineRepository: pipelineRepository, ciArtifactRepository: ciArtifactRepository, + cdWorkflowRepository: cdWorkflowRepository, } return impl, nil } -func (impl *WorkflowEventProcessorImpl) SubscribeCdStageCompleteEvent() error { +func (impl *WorkflowEventProcessorImpl) SubscribeCDStageCompleteEvent() error { callback := func(msg *model.PubSubMsg) { cdStageCompleteEvent := bean.CdStageCompleteEvent{} err := json.Unmarshal([]byte(msg.Data), &cdStageCompleteEvent) @@ -247,3 +264,141 @@ func (impl *WorkflowEventProcessorImpl) SubscribeHibernateBulkAction() error { err := impl.pubSubClient.Subscribe(pubsub.BULK_HIBERNATE_TOPIC, callback, loggerFunc) return err } + +func (impl *WorkflowEventProcessorImpl) SubscribeCIWorkflowStatusUpdate() error { + callback := func(msg *model.PubSubMsg) { + wfStatus := v1alpha1.WorkflowStatus{} + err := json.Unmarshal([]byte(msg.Data), &wfStatus) + if err != nil { + impl.logger.Errorw("error while unmarshalling wf status update", "err", err, "msg", msg.Data) + return + } + + err = impl.ciHandler.CheckAndReTriggerCI(wfStatus) + if err != nil { + impl.logger.Errorw("error in checking and re triggering ci", "err", err) + //don't return as we have to update the workflow status + } + + _, err = impl.ciHandler.UpdateWorkflow(wfStatus) + if err != nil { + impl.logger.Errorw("error on update workflow status", "err", err, "msg", msg.Data) + return + } + + } + + // add required logging here + var loggerFunc pubsub.LoggerFunc = func(msg model.PubSubMsg) (string, []interface{}) { + wfStatus := v1alpha1.WorkflowStatus{} + err := json.Unmarshal([]byte(msg.Data), &wfStatus) + if err != nil { + return "error while unmarshalling wf status update", []interface{}{"err", err, "msg", msg.Data} + } + + workflowName, status, _, message, _, _ := pipeline.ExtractWorkflowStatus(wfStatus) + return "got message for ci workflow status update ", []interface{}{"workflowName", workflowName, "status", status, "message", message} + } + + err := impl.pubSubClient.Subscribe(pubsub.WORKFLOW_STATUS_UPDATE_TOPIC, callback, loggerFunc) + + if err != nil { + impl.logger.Error("err", err) + return err + } + return nil +} + +func (impl *WorkflowEventProcessorImpl) SubscribeCDWorkflowStatusUpdate() error { + callback := func(msg *model.PubSubMsg) { + wfStatus := v1alpha1.WorkflowStatus{} + err := json.Unmarshal([]byte(msg.Data), &wfStatus) + if err != nil { + impl.logger.Error("Error while unmarshalling wfStatus json object", "error", err) + return + } + + wfrId, wfrStatus, err := impl.cdHandler.UpdateWorkflow(wfStatus) + impl.logger.Debugw("UpdateWorkflow", "wfrId", wfrId, "wfrStatus", wfrStatus) + if err != nil { + impl.logger.Error("err", err) + return + } + + wfr, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(wfrId) + if err != nil { + impl.logger.Errorw("could not get wf runner", "err", err) + return + } + if wfrStatus == string(v1alpha1.NodeFailed) || wfrStatus == string(v1alpha1.NodeError) { + if len(wfr.ImagePathReservationIds) > 0 { + err := impl.cdHandler.DeactivateImageReservationPathsOnFailure(wfr.ImagePathReservationIds) + if err != nil { + impl.logger.Errorw("error in removing image path reservation ") + } + } + } + if wfrStatus == string(v1alpha1.NodeSucceeded) || wfrStatus == string(v1alpha1.NodeFailed) || wfrStatus == string(v1alpha1.NodeError) { + eventType := util.EventType(0) + if wfrStatus == string(v1alpha1.NodeSucceeded) { + eventType = util.Success + } else if wfrStatus == string(v1alpha1.NodeFailed) || wfrStatus == string(v1alpha1.NodeError) { + eventType = util.Fail + } + + if wfr != nil && executors.CheckIfReTriggerRequired(wfrStatus, wfStatus.Message, wfr.Status) { + err = impl.cdHandler.HandleCdStageReTrigger(wfr) + if err != nil { + //check if this log required or not + impl.logger.Errorw("error in HandleCdStageReTrigger", "error", err) + } + } + + if wfr.WorkflowType == bean2.CD_WORKFLOW_TYPE_PRE || wfr.WorkflowType == bean2.CD_WORKFLOW_TYPE_POST { + event := impl.eventFactory.Build(eventType, &wfr.CdWorkflow.PipelineId, wfr.CdWorkflow.Pipeline.AppId, &wfr.CdWorkflow.Pipeline.EnvironmentId, util.CD) + impl.logger.Debugw("event pre stage", "event", event) + event = impl.eventFactory.BuildExtraCDData(event, wfr, 0, wfr.WorkflowType) + _, evtErr := impl.eventClient.WriteNotificationEvent(event) + if evtErr != nil { + impl.logger.Errorw("CD stage post fail or success event unable to sent", "error", evtErr) + } + } + + if wfr.WorkflowType == bean2.CD_WORKFLOW_TYPE_PRE { + event := impl.eventFactory.Build(eventType, &wfr.CdWorkflow.PipelineId, wfr.CdWorkflow.Pipeline.AppId, &wfr.CdWorkflow.Pipeline.EnvironmentId, util.CD) + impl.logger.Debugw("event pre stage", "event", event) + event = impl.eventFactory.BuildExtraCDData(event, wfr, 0, bean2.CD_WORKFLOW_TYPE_PRE) + _, evtErr := impl.eventClient.WriteNotificationEvent(event) + if evtErr != nil { + impl.logger.Errorw("CD stage post fail or success event unable to sent", "error", evtErr) + } + } else if wfr.WorkflowType == bean2.CD_WORKFLOW_TYPE_POST { + event := impl.eventFactory.Build(eventType, &wfr.CdWorkflow.PipelineId, wfr.CdWorkflow.Pipeline.AppId, &wfr.CdWorkflow.Pipeline.EnvironmentId, util.CD) + impl.logger.Debugw("event post stage", "event", event) + event = impl.eventFactory.BuildExtraCDData(event, wfr, 0, bean2.CD_WORKFLOW_TYPE_POST) + _, evtErr := impl.eventClient.WriteNotificationEvent(event) + if evtErr != nil { + impl.logger.Errorw("CD stage post fail or success event not sent", "error", evtErr) + } + } + } + } + + // add required logging here + var loggerFunc pubsub.LoggerFunc = func(msg model.PubSubMsg) (string, []interface{}) { + wfStatus := v1alpha1.WorkflowStatus{} + err := json.Unmarshal([]byte(msg.Data), &wfStatus) + if err != nil { + return "error while unmarshalling wfStatus json object", []interface{}{"error", err} + } + workflowName, status, _, message, _, _ := pipeline.ExtractWorkflowStatus(wfStatus) + return "got message for cd workflow status", []interface{}{"workflowName", workflowName, "status", status, "message", message} + } + + err := impl.pubSubClient.Subscribe(pubsub.CD_WORKFLOW_STATUS_UPDATE, callback, loggerFunc) + if err != nil { + impl.logger.Error("err", err) + return err + } + return nil +} diff --git a/wire_gen.go b/wire_gen.go index 493989bccd..ad83951a29 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -653,7 +653,6 @@ func InitializeApp() (*App, error) { teamRestHandlerImpl := team2.NewTeamRestHandlerImpl(sugaredLogger, teamServiceImpl, userServiceImpl, enforcerImpl, validate, userAuthServiceImpl, deleteServiceExtendedImpl) teamRouterImpl := team2.NewTeamRouterImpl(teamRestHandlerImpl) gitWebhookHandlerImpl := pubsub.NewGitWebhookHandler(sugaredLogger, pubSubClientServiceImpl, gitWebhookServiceImpl) - workflowStatusUpdateHandlerImpl := pubsub.NewWorkflowStatusUpdateHandlerImpl(sugaredLogger, pubSubClientServiceImpl, ciHandlerImpl, cdHandlerImpl, eventSimpleFactoryImpl, eventRESTClientImpl, cdWorkflowRepositoryImpl) applicationStatusHandlerImpl := pubsub.NewApplicationStatusHandlerImpl(sugaredLogger, pubSubClientServiceImpl, appServiceImpl, workflowDagExecutorImpl, installedAppServiceImpl, appStoreDeploymentServiceImpl, pipelineBuilderImpl, pipelineRepositoryImpl, installedAppRepositoryImpl, gitOpsConfigReadServiceImpl) roleGroupServiceImpl := user.NewRoleGroupServiceImpl(userAuthRepositoryImpl, sugaredLogger, userRepositoryImpl, roleGroupRepositoryImpl, userCommonServiceImpl) userRestHandlerImpl := user2.NewUserRestHandlerImpl(userServiceImpl, validate, sugaredLogger, enforcerImpl, roleGroupServiceImpl, userCommonServiceImpl) @@ -837,11 +836,11 @@ func InitializeApp() (*App, error) { return nil, err } proxyRouterImpl := proxy.NewProxyRouterImpl(sugaredLogger, proxyConfig, enforcerImpl) - muxRouter := router.NewMuxRouter(sugaredLogger, pipelineTriggerRouterImpl, pipelineConfigRouterImpl, appListingRouterImpl, environmentRouterImpl, clusterRouterImpl, webhookRouterImpl, userAuthRouterImpl, applicationRouterImpl, cdRouterImpl, gitProviderRouterImpl, gitHostRouterImpl, dockerRegRouterImpl, notificationRouterImpl, teamRouterImpl, gitWebhookHandlerImpl, workflowStatusUpdateHandlerImpl, applicationStatusHandlerImpl, ciEventHandlerImpl, pubSubClientServiceImpl, userRouterImpl, chartRefRouterImpl, configMapRouterImpl, appStoreRouterImpl, chartRepositoryRouterImpl, releaseMetricsRouterImpl, deploymentGroupRouterImpl, batchOperationRouterImpl, chartGroupRouterImpl, imageScanRouterImpl, policyRouterImpl, gitOpsConfigRouterImpl, dashboardRouterImpl, attributesRouterImpl, userAttributesRouterImpl, commonRouterImpl, grafanaRouterImpl, ssoLoginRouterImpl, telemetryRouterImpl, telemetryEventClientImplExtended, bulkUpdateRouterImpl, webhookListenerRouterImpl, appRouterImpl, coreAppRouterImpl, helmAppRouterImpl, k8sApplicationRouterImpl, pProfRouterImpl, deploymentConfigRouterImpl, dashboardTelemetryRouterImpl, commonDeploymentRouterImpl, externalLinkRouterImpl, globalPluginRouterImpl, moduleRouterImpl, serverRouterImpl, apiTokenRouterImpl, cdApplicationStatusUpdateHandlerImpl, k8sCapacityRouterImpl, webhookHelmRouterImpl, globalCMCSRouterImpl, userTerminalAccessRouterImpl, jobRouterImpl, ciStatusUpdateCronImpl, resourceGroupingRouterImpl, rbacRoleRouterImpl, scopedVariableRouterImpl, ciTriggerCronImpl, proxyRouterImpl) + muxRouter := router.NewMuxRouter(sugaredLogger, pipelineTriggerRouterImpl, pipelineConfigRouterImpl, appListingRouterImpl, environmentRouterImpl, clusterRouterImpl, webhookRouterImpl, userAuthRouterImpl, applicationRouterImpl, cdRouterImpl, gitProviderRouterImpl, gitHostRouterImpl, dockerRegRouterImpl, notificationRouterImpl, teamRouterImpl, gitWebhookHandlerImpl, applicationStatusHandlerImpl, ciEventHandlerImpl, pubSubClientServiceImpl, userRouterImpl, chartRefRouterImpl, configMapRouterImpl, appStoreRouterImpl, chartRepositoryRouterImpl, releaseMetricsRouterImpl, deploymentGroupRouterImpl, batchOperationRouterImpl, chartGroupRouterImpl, imageScanRouterImpl, policyRouterImpl, gitOpsConfigRouterImpl, dashboardRouterImpl, attributesRouterImpl, userAttributesRouterImpl, commonRouterImpl, grafanaRouterImpl, ssoLoginRouterImpl, telemetryRouterImpl, telemetryEventClientImplExtended, bulkUpdateRouterImpl, webhookListenerRouterImpl, appRouterImpl, coreAppRouterImpl, helmAppRouterImpl, k8sApplicationRouterImpl, pProfRouterImpl, deploymentConfigRouterImpl, dashboardTelemetryRouterImpl, commonDeploymentRouterImpl, externalLinkRouterImpl, globalPluginRouterImpl, moduleRouterImpl, serverRouterImpl, apiTokenRouterImpl, cdApplicationStatusUpdateHandlerImpl, k8sCapacityRouterImpl, webhookHelmRouterImpl, globalCMCSRouterImpl, userTerminalAccessRouterImpl, jobRouterImpl, ciStatusUpdateCronImpl, resourceGroupingRouterImpl, rbacRoleRouterImpl, scopedVariableRouterImpl, ciTriggerCronImpl, proxyRouterImpl) loggingMiddlewareImpl := util4.NewLoggingMiddlewareImpl(userServiceImpl) cdWorkflowServiceImpl := cd.NewCdWorkflowServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) cdWorkflowRunnerServiceImpl := cd.NewCdWorkflowRunnerServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) - workflowEventProcessorImpl, err := in.NewWorkflowEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowServiceImpl, cdWorkflowRunnerServiceImpl, workflowDagExecutorImpl, argoUserServiceImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl) + workflowEventProcessorImpl, err := in.NewWorkflowEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowServiceImpl, cdWorkflowRunnerServiceImpl, workflowDagExecutorImpl, argoUserServiceImpl, ciHandlerImpl, cdHandlerImpl, eventSimpleFactoryImpl, eventRESTClientImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, cdWorkflowRepositoryImpl) if err != nil { return nil, err } From e7f2f157802a0629dc44fb05d52e394eaad978fe Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Fri, 2 Feb 2024 13:42:09 +0530 Subject: [PATCH 54/64] removed old refactored code --- .../in/WorkflowEventProcessorService.go | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/pkg/eventProcessor/in/WorkflowEventProcessorService.go b/pkg/eventProcessor/in/WorkflowEventProcessorService.go index 51cc7576ab..7faf6ea1a6 100644 --- a/pkg/eventProcessor/in/WorkflowEventProcessorService.go +++ b/pkg/eventProcessor/in/WorkflowEventProcessorService.go @@ -363,24 +363,6 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDWorkflowStatusUpdate() error impl.logger.Errorw("CD stage post fail or success event unable to sent", "error", evtErr) } } - - if wfr.WorkflowType == bean2.CD_WORKFLOW_TYPE_PRE { - event := impl.eventFactory.Build(eventType, &wfr.CdWorkflow.PipelineId, wfr.CdWorkflow.Pipeline.AppId, &wfr.CdWorkflow.Pipeline.EnvironmentId, util.CD) - impl.logger.Debugw("event pre stage", "event", event) - event = impl.eventFactory.BuildExtraCDData(event, wfr, 0, bean2.CD_WORKFLOW_TYPE_PRE) - _, evtErr := impl.eventClient.WriteNotificationEvent(event) - if evtErr != nil { - impl.logger.Errorw("CD stage post fail or success event unable to sent", "error", evtErr) - } - } else if wfr.WorkflowType == bean2.CD_WORKFLOW_TYPE_POST { - event := impl.eventFactory.Build(eventType, &wfr.CdWorkflow.PipelineId, wfr.CdWorkflow.Pipeline.AppId, &wfr.CdWorkflow.Pipeline.EnvironmentId, util.CD) - impl.logger.Debugw("event post stage", "event", event) - event = impl.eventFactory.BuildExtraCDData(event, wfr, 0, bean2.CD_WORKFLOW_TYPE_POST) - _, evtErr := impl.eventClient.WriteNotificationEvent(event) - if evtErr != nil { - impl.logger.Errorw("CD stage post fail or success event not sent", "error", evtErr) - } - } } } From 84de9ddc4ac1cd79669b9704e8e0744511330f7a Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Fri, 2 Feb 2024 17:12:22 +0530 Subject: [PATCH 55/64] wip - extracted k8s op method from workflowDAG part 1 --- api/restHandler/PipelineTriggerRestHandler.go | 11 +++- .../deployedApp/DeployedAppService.go | 56 +++++++++++++++++++ pkg/deployment/deployedApp/bean/bean.go | 10 ++++ .../deployedApp/wire_deployedApp.go | 10 ++++ .../trigger/wire_deploymenTrigger.go | 10 ++++ pkg/deployment/wire_deployment.go | 6 +- pkg/k8s/K8sCommonService.go | 3 +- pkg/pipeline/WorkflowDagExecutor.go | 35 ------------ wire_gen.go | 4 +- 9 files changed, 103 insertions(+), 42 deletions(-) create mode 100644 pkg/deployment/deployedApp/DeployedAppService.go create mode 100644 pkg/deployment/deployedApp/bean/bean.go create mode 100644 pkg/deployment/deployedApp/wire_deployedApp.go create mode 100644 pkg/deployment/trigger/wire_deploymenTrigger.go diff --git a/api/restHandler/PipelineTriggerRestHandler.go b/api/restHandler/PipelineTriggerRestHandler.go index f019462e65..a943bb4e3c 100644 --- a/api/restHandler/PipelineTriggerRestHandler.go +++ b/api/restHandler/PipelineTriggerRestHandler.go @@ -21,6 +21,8 @@ import ( "context" "encoding/json" "fmt" + "github.com/devtron-labs/devtron/pkg/deployment/deployedApp" + bean2 "github.com/devtron-labs/devtron/pkg/deployment/deployedApp/bean" "net/http" "strconv" @@ -63,12 +65,14 @@ type PipelineTriggerRestHandlerImpl struct { deploymentGroupService deploymentGroup.DeploymentGroupService argoUserService argo.ArgoUserService deploymentConfigService pipeline.DeploymentConfigService + deployedAppService deployedApp.DeployedAppService } func NewPipelineRestHandler(appService app.AppService, userAuthService user.UserService, validator *validator.Validate, enforcer casbin.Enforcer, teamService team.TeamService, logger *zap.SugaredLogger, enforcerUtil rbac.EnforcerUtil, workflowDagExecutor pipeline.WorkflowDagExecutor, deploymentGroupService deploymentGroup.DeploymentGroupService, - argoUserService argo.ArgoUserService, deploymentConfigService pipeline.DeploymentConfigService) *PipelineTriggerRestHandlerImpl { + argoUserService argo.ArgoUserService, deploymentConfigService pipeline.DeploymentConfigService, + deployedAppService deployedApp.DeployedAppService) *PipelineTriggerRestHandlerImpl { pipelineHandler := &PipelineTriggerRestHandlerImpl{ appService: appService, userAuthService: userAuthService, @@ -81,6 +85,7 @@ func NewPipelineRestHandler(appService app.AppService, userAuthService user.User deploymentGroupService: deploymentGroupService, argoUserService: argoUserService, deploymentConfigService: deploymentConfigService, + deployedAppService: deployedAppService, } return pipelineHandler } @@ -150,7 +155,7 @@ func (handler PipelineTriggerRestHandlerImpl) RotatePods(w http.ResponseWriter, common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) return } - var podRotateRequest pipeline.PodRotateRequest + var podRotateRequest bean2.PodRotateRequest err = decoder.Decode(&podRotateRequest) if err != nil { handler.logger.Errorw("request err, RotatePods", "err", err, "payload", podRotateRequest) @@ -176,7 +181,7 @@ func (handler PipelineTriggerRestHandlerImpl) RotatePods(w http.ResponseWriter, common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden) return } - rotatePodResponse, err := handler.workflowDagExecutor.RotatePods(r.Context(), &podRotateRequest) + rotatePodResponse, err := handler.deployedAppService.RotatePods(r.Context(), &podRotateRequest) if err != nil { handler.logger.Errorw("service err, RotatePods", "err", err, "payload", podRotateRequest) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) diff --git a/pkg/deployment/deployedApp/DeployedAppService.go b/pkg/deployment/deployedApp/DeployedAppService.go new file mode 100644 index 0000000000..e4f0549dd9 --- /dev/null +++ b/pkg/deployment/deployedApp/DeployedAppService.go @@ -0,0 +1,56 @@ +package deployedApp + +import ( + "context" + util5 "github.com/devtron-labs/common-lib/utils/k8s" + "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/deployment/deployedApp/bean" + "github.com/devtron-labs/devtron/pkg/k8s" + "go.uber.org/zap" +) + +type DeployedAppService interface { + RotatePods(ctx context.Context, podRotateRequest *bean.PodRotateRequest) (*k8s.RotatePodResponse, error) +} + +type DeployedAppServiceImpl struct { + logger *zap.SugaredLogger + k8sCommonService k8s.K8sCommonService + envRepository repository.EnvironmentRepository +} + +func NewDeployedAppServiceImpl(logger *zap.SugaredLogger, + k8sCommonService k8s.K8sCommonService, + envRepository repository.EnvironmentRepository) *DeployedAppServiceImpl { + return &DeployedAppServiceImpl{ + logger: logger, + k8sCommonService: k8sCommonService, + envRepository: envRepository, + } +} + +func (impl *DeployedAppServiceImpl) RotatePods(ctx context.Context, podRotateRequest *bean.PodRotateRequest) (*k8s.RotatePodResponse, error) { + impl.logger.Infow("rotate pod request", "payload", podRotateRequest) + //extract cluster id and namespace from env id + environmentId := podRotateRequest.EnvironmentId + environment, err := impl.envRepository.FindById(environmentId) + if err != nil { + impl.logger.Errorw("error occurred while fetching env details", "envId", environmentId, "err", err) + return nil, err + } + var resourceIdentifiers []util5.ResourceIdentifier + for _, resourceIdentifier := range podRotateRequest.ResourceIdentifiers { + resourceIdentifier.Namespace = environment.Namespace + resourceIdentifiers = append(resourceIdentifiers, resourceIdentifier) + } + rotatePodRequest := &k8s.RotatePodRequest{ + ClusterId: environment.ClusterId, + Resources: resourceIdentifiers, + } + response, err := impl.k8sCommonService.RotatePods(ctx, rotatePodRequest) + if err != nil { + return nil, err + } + //TODO KB: make entry in cd workflow runner + return response, nil +} diff --git a/pkg/deployment/deployedApp/bean/bean.go b/pkg/deployment/deployedApp/bean/bean.go new file mode 100644 index 0000000000..ba1865b3c1 --- /dev/null +++ b/pkg/deployment/deployedApp/bean/bean.go @@ -0,0 +1,10 @@ +package bean + +import util5 "github.com/devtron-labs/common-lib/utils/k8s" + +type PodRotateRequest struct { + AppId int `json:"appId" validate:"required"` + EnvironmentId int `json:"environmentId" validate:"required"` + ResourceIdentifiers []util5.ResourceIdentifier `json:"resources" validate:"required"` + UserId int32 `json:"-"` +} diff --git a/pkg/deployment/deployedApp/wire_deployedApp.go b/pkg/deployment/deployedApp/wire_deployedApp.go new file mode 100644 index 0000000000..4c8dd05a07 --- /dev/null +++ b/pkg/deployment/deployedApp/wire_deployedApp.go @@ -0,0 +1,10 @@ +package deployedApp + +import ( + "github.com/google/wire" +) + +var DeployedAppWireSet = wire.NewSet( + NewDeployedAppServiceImpl, + wire.Bind(new(DeployedAppService), new(*DeployedAppServiceImpl)), +) diff --git a/pkg/deployment/trigger/wire_deploymenTrigger.go b/pkg/deployment/trigger/wire_deploymenTrigger.go new file mode 100644 index 0000000000..51d02da2f7 --- /dev/null +++ b/pkg/deployment/trigger/wire_deploymenTrigger.go @@ -0,0 +1,10 @@ +package trigger + +import ( + "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps" + "github.com/google/wire" +) + +var DeploymentTriggerWireSet = wire.NewSet( + devtronApps.DevtronAppsDeployTriggerWireSet, +) diff --git a/pkg/deployment/wire_deployment.go b/pkg/deployment/wire_deployment.go index 610b6a8ea4..d3d568a337 100644 --- a/pkg/deployment/wire_deployment.go +++ b/pkg/deployment/wire_deployment.go @@ -1,9 +1,10 @@ package deployment import ( + "github.com/devtron-labs/devtron/pkg/deployment/deployedApp" "github.com/devtron-labs/devtron/pkg/deployment/gitOps" "github.com/devtron-labs/devtron/pkg/deployment/manifest" - "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps" + "github.com/devtron-labs/devtron/pkg/deployment/trigger" "github.com/google/wire" ) @@ -12,5 +13,6 @@ import ( var DeploymentWireSet = wire.NewSet( gitOps.GitOpsWireSet, manifest.DeploymentManifestWireSet, - devtronApps.DevtronAppsDeployTriggerWireSet, + trigger.DeploymentTriggerWireSet, + deployedApp.DeployedAppWireSet, ) diff --git a/pkg/k8s/K8sCommonService.go b/pkg/k8s/K8sCommonService.go index 813d64b7a3..c8c167c379 100644 --- a/pkg/k8s/K8sCommonService.go +++ b/pkg/k8s/K8sCommonService.go @@ -247,7 +247,8 @@ func (impl *K8sCommonServiceImpl) RotatePods(ctx context.Context, request *Rotat namespace := resourceIdentifier.Namespace resourceKind := groupVersionKind.Kind // validate one of deployment, statefulset, daemonSet, Rollout - if resourceKind != k8sCommonBean.DeploymentKind && resourceKind != k8sCommonBean.StatefulSetKind && resourceKind != k8sCommonBean.DaemonSetKind && resourceKind != k8sCommonBean.K8sClusterResourceRolloutKind { + if resourceKind != k8sCommonBean.DeploymentKind && resourceKind != k8sCommonBean.StatefulSetKind && + resourceKind != k8sCommonBean.DaemonSetKind && resourceKind != k8sCommonBean.K8sClusterResourceRolloutKind { impl.logger.Errorf("restarting not supported for kind %s name %s", resourceKind, resourceIdentifier.Name) containsError = true resourceResponse.ErrorResponse = k8s.RestartingNotSupported diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index 79443531c8..e7f5dab239 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -36,7 +36,6 @@ import ( blob_storage "github.com/devtron-labs/common-lib/blob-storage" "github.com/devtron-labs/common-lib/pubsub-lib/model" - util5 "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/common-lib/utils/k8s/health" "github.com/devtron-labs/devtron/client/argocdServer/application" gitSensorClient "github.com/devtron-labs/devtron/client/gitSensor" @@ -97,7 +96,6 @@ type WorkflowDagExecutor interface { TriggerBulkHibernateAsync(request StopDeploymentGroupRequest, ctx context.Context) (interface{}, error) StopStartApp(triggerContext TriggerContext, stopRequest *StopAppRequest) (int, error) - RotatePods(ctx context.Context, podRotateRequest *PodRotateRequest) (*k8s.RotatePodResponse, error) MarkCurrentDeploymentFailed(runner *pipelineConfig.CdWorkflowRunner, releaseErr error, triggeredBy int32) error UpdateWorkflowRunnerStatusForDeployment(appIdentifier *client2.AppIdentifier, wfr *pipelineConfig.CdWorkflowRunner, skipReleaseNotFound bool) bool @@ -2099,39 +2097,6 @@ type StopDeploymentGroupRequest struct { RequestType RequestType `json:"requestType" validate:"oneof=START STOP"` } -type PodRotateRequest struct { - AppId int `json:"appId" validate:"required"` - EnvironmentId int `json:"environmentId" validate:"required"` - UserId int32 `json:"-"` - ResourceIdentifiers []util5.ResourceIdentifier `json:"resources" validate:"required"` -} - -func (impl *WorkflowDagExecutorImpl) RotatePods(ctx context.Context, podRotateRequest *PodRotateRequest) (*k8s.RotatePodResponse, error) { - impl.logger.Infow("rotate pod request", "payload", podRotateRequest) - //extract cluster id and namespace from env id - environmentId := podRotateRequest.EnvironmentId - environment, err := impl.envRepository.FindById(environmentId) - if err != nil { - impl.logger.Errorw("error occurred while fetching env details", "envId", environmentId, "err", err) - return nil, err - } - var resourceIdentifiers []util5.ResourceIdentifier - for _, resourceIdentifier := range podRotateRequest.ResourceIdentifiers { - resourceIdentifier.Namespace = environment.Namespace - resourceIdentifiers = append(resourceIdentifiers, resourceIdentifier) - } - rotatePodRequest := &k8s.RotatePodRequest{ - ClusterId: environment.ClusterId, - Resources: resourceIdentifiers, - } - response, err := impl.k8sCommonService.RotatePods(ctx, rotatePodRequest) - if err != nil { - return nil, err - } - //TODO KB: make entry in cd workflow runner - return response, nil -} - func (impl *WorkflowDagExecutorImpl) StopStartApp(triggerContext TriggerContext, stopRequest *StopAppRequest) (int, error) { pipelines, err := impl.pipelineRepository.FindActiveByAppIdAndEnvironmentId(stopRequest.AppId, stopRequest.EnvironmentId) if err != nil { diff --git a/wire_gen.go b/wire_gen.go index ad83951a29..913f9e7bb5 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -105,6 +105,7 @@ import ( "github.com/devtron-labs/devtron/pkg/clusterTerminalAccess" "github.com/devtron-labs/devtron/pkg/commonService" delete2 "github.com/devtron-labs/devtron/pkg/delete" + "github.com/devtron-labs/devtron/pkg/deployment/deployedApp" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" "github.com/devtron-labs/devtron/pkg/deployment/manifest" @@ -479,7 +480,8 @@ func InitializeApp() (*App, error) { deploymentGroupAppRepositoryImpl := repository.NewDeploymentGroupAppRepositoryImpl(sugaredLogger, db) deploymentGroupServiceImpl := deploymentGroup.NewDeploymentGroupServiceImpl(appRepositoryImpl, sugaredLogger, pipelineRepositoryImpl, ciPipelineRepositoryImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, deploymentGroupAppRepositoryImpl, ciArtifactRepositoryImpl, appWorkflowRepositoryImpl, workflowEventPublishServiceImpl) deploymentConfigServiceImpl := pipeline.NewDeploymentConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, pipelineRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, configMapHistoryServiceImpl, scopedVariableCMCSManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) - pipelineTriggerRestHandlerImpl := restHandler.NewPipelineRestHandler(appServiceImpl, userServiceImpl, validate, enforcerImpl, teamServiceImpl, sugaredLogger, enforcerUtilImpl, workflowDagExecutorImpl, deploymentGroupServiceImpl, argoUserServiceImpl, deploymentConfigServiceImpl) + deployedAppServiceImpl := deployedApp.NewDeployedAppServiceImpl(sugaredLogger, k8sCommonServiceImpl, environmentRepositoryImpl) + pipelineTriggerRestHandlerImpl := restHandler.NewPipelineRestHandler(appServiceImpl, userServiceImpl, validate, enforcerImpl, teamServiceImpl, sugaredLogger, enforcerUtilImpl, workflowDagExecutorImpl, deploymentGroupServiceImpl, argoUserServiceImpl, deploymentConfigServiceImpl, deployedAppServiceImpl) sseSSE := sse.NewSSE() pipelineTriggerRouterImpl := router.NewPipelineTriggerRouter(pipelineTriggerRestHandlerImpl, sseSSE) appListingRepositoryQueryBuilder := helper.NewAppListingRepositoryQueryBuilder(sugaredLogger) From 7c11b67b8d1b936e31b6fccf6f05fd9376431571 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Fri, 2 Feb 2024 17:54:55 +0530 Subject: [PATCH 56/64] extracted artifact logic from workflowDAG --- .../manifest/ManifestCreationService.go | 24 ++++++ pkg/pipeline/WorkflowDagExecutor.go | 85 +++---------------- 2 files changed, 34 insertions(+), 75 deletions(-) diff --git a/pkg/deployment/manifest/ManifestCreationService.go b/pkg/deployment/manifest/ManifestCreationService.go index fece14b7f4..f53e587fb4 100644 --- a/pkg/deployment/manifest/ManifestCreationService.go +++ b/pkg/deployment/manifest/ManifestCreationService.go @@ -21,6 +21,7 @@ import ( repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" bean3 "github.com/devtron-labs/devtron/pkg/deployment/manifest/bean" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" "github.com/devtron-labs/devtron/pkg/dockerRegistry" "github.com/devtron-labs/devtron/pkg/imageDigestPolicy" @@ -47,6 +48,10 @@ import ( ) type ManifestCreationService interface { + BuildManifestForTrigger(overrideRequest *bean.ValuesOverrideRequest, triggeredAt time.Time, + ctx context.Context) (valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, err error) + + //TODO: remove below method GetValuesOverrideForTrigger(overrideRequest *bean.ValuesOverrideRequest, triggeredAt time.Time, ctx context.Context) (*app.ValuesOverrideResponse, error) } @@ -60,6 +65,7 @@ type ManifestCreationServiceImpl struct { imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService mergeUtil *util.MergeUtil appCrudOperationService app.AppCrudOperationService + deploymentTemplateService deploymentTemplate.DeploymentTemplateService acdClient application2.ServiceClient //TODO: replace with argoClientWrapperService @@ -85,6 +91,7 @@ func NewManifestCreationServiceImpl(logger *zap.SugaredLogger, imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService, mergeUtil *util.MergeUtil, appCrudOperationService app.AppCrudOperationService, + deploymentTemplateService deploymentTemplate.DeploymentTemplateService, acdClient application2.ServiceClient, configMapHistoryRepository repository3.ConfigMapHistoryRepository, configMapRepository chartConfig.ConfigMapRepository, @@ -107,6 +114,7 @@ func NewManifestCreationServiceImpl(logger *zap.SugaredLogger, imageDigestPolicyService: imageDigestPolicyService, mergeUtil: mergeUtil, appCrudOperationService: appCrudOperationService, + deploymentTemplateService: deploymentTemplateService, configMapRepository: configMapRepository, acdClient: acdClient, configMapHistoryRepository: configMapHistoryRepository, @@ -122,6 +130,22 @@ func NewManifestCreationServiceImpl(logger *zap.SugaredLogger, } } +func (impl *ManifestCreationServiceImpl) BuildManifestForTrigger(overrideRequest *bean.ValuesOverrideRequest, triggeredAt time.Time, + ctx context.Context) (valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, err error) { + valuesOverrideResponse = &app.ValuesOverrideResponse{} + valuesOverrideResponse, err = impl.GetValuesOverrideForTrigger(overrideRequest, triggeredAt, ctx) + if err != nil { + impl.logger.Errorw("error in fetching values for trigger", "err", err) + return valuesOverrideResponse, "", err + } + builtChartPath, err = impl.deploymentTemplateService.BuildChartAndGetPath(overrideRequest.AppName, valuesOverrideResponse.EnvOverride, ctx) + if err != nil { + impl.logger.Errorw("error in parsing reference chart", "err", err) + return valuesOverrideResponse, "", err + } + return valuesOverrideResponse, builtChartPath, err +} + func (impl *ManifestCreationServiceImpl) GetValuesOverrideForTrigger(overrideRequest *bean.ValuesOverrideRequest, triggeredAt time.Time, ctx context.Context) (*app.ValuesOverrideResponse, error) { if overrideRequest.DeploymentType == models.DEPLOYMENTTYPE_UNKNOWN { overrideRequest.DeploymentType = models.DEPLOYMENTTYPE_DEPLOY diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index e7f5dab239..57e18f5a60 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -23,6 +23,7 @@ import ( "fmt" bean6 "github.com/devtron-labs/devtron/api/helm-app/bean" client2 "github.com/devtron-labs/devtron/api/helm-app/service" + "github.com/devtron-labs/devtron/pkg/build/artifacts" "github.com/devtron-labs/devtron/pkg/deployment/manifest" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps" @@ -42,7 +43,6 @@ import ( "github.com/devtron-labs/devtron/pkg/app/status" "github.com/devtron-labs/devtron/pkg/auth/user" "github.com/devtron-labs/devtron/pkg/imageDigestPolicy" - "github.com/devtron-labs/devtron/pkg/k8s" bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" repository4 "github.com/devtron-labs/devtron/pkg/pipeline/repository" "github.com/devtron-labs/devtron/pkg/pipeline/types" @@ -130,7 +130,6 @@ type WorkflowDagExecutorImpl struct { ciWorkflowRepository pipelineConfig.CiWorkflowRepository appLabelRepository pipelineConfig.AppLabelRepository gitSensorGrpcClient gitSensorClient.Client - k8sCommonService k8s.K8sCommonService pipelineStageService PipelineStageService config *types.CdConfig appServiceConfig *app.AppServiceConfig @@ -155,6 +154,7 @@ type WorkflowDagExecutorImpl struct { workflowEventPublishService out.WorkflowEventPublishService manifestCreationService manifest.ManifestCreationService deploymentTemplateService deploymentTemplate.DeploymentTemplateService + commonArtifactService artifacts.CommonArtifactService } const ( @@ -218,7 +218,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi CiTemplateRepository pipelineConfig.CiTemplateRepository, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, appLabelRepository pipelineConfig.AppLabelRepository, gitSensorGrpcClient gitSensorClient.Client, - pipelineStageService PipelineStageService, k8sCommonService k8s.K8sCommonService, + pipelineStageService PipelineStageService, globalPluginService plugin.GlobalPluginService, pluginInputVariableParser PluginInputVariableParser, scopedVariableManager variables.ScopedVariableCMCSManager, @@ -231,7 +231,8 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi deployedConfigurationHistoryService history2.DeployedConfigurationHistoryService, workflowEventPublishService out.WorkflowEventPublishService, manifestCreationService manifest.ManifestCreationService, - deploymentTemplateService deploymentTemplate.DeploymentTemplateService) *WorkflowDagExecutorImpl { + deploymentTemplateService deploymentTemplate.DeploymentTemplateService, + commonArtifactService artifacts.CommonArtifactService) *WorkflowDagExecutorImpl { wde := &WorkflowDagExecutorImpl{logger: Logger, pipelineRepository: pipelineRepository, cdWorkflowRepository: cdWorkflowRepository, @@ -258,7 +259,6 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi ciWorkflowRepository: ciWorkflowRepository, appLabelRepository: appLabelRepository, gitSensorGrpcClient: gitSensorGrpcClient, - k8sCommonService: k8sCommonService, pipelineStageService: pipelineStageService, scopedVariableManager: scopedVariableManager, globalPluginService: globalPluginService, @@ -277,6 +277,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi workflowEventPublishService: workflowEventPublishService, manifestCreationService: manifestCreationService, deploymentTemplateService: deploymentTemplateService, + commonArtifactService: commonArtifactService, } config, err := types.GetCdConfig() if err != nil { @@ -551,7 +552,7 @@ func (impl *WorkflowDagExecutorImpl) processDevtronAsyncHelmInstallRequest(CDAsy return } // build merged values and save PCO history for the release - valuesOverrideResponse, builtChartPath, err := impl.BuildManifestForTrigger(overrideRequest, CDAsyncInstallNatsMessage.TriggeredAt, ctx) + valuesOverrideResponse, builtChartPath, err := impl.manifestCreationService.BuildManifestForTrigger(overrideRequest, CDAsyncInstallNatsMessage.TriggeredAt, ctx) if err != nil { return } @@ -796,7 +797,7 @@ func (impl *WorkflowDagExecutorImpl) HandlePreStageSuccessEvent(triggerContext T impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", ciArtifact.Id) } } - PreCDArtifacts, err := impl.SavePluginArtifacts(ciArtifact, cdStageCompleteEvent.PluginRegistryArtifactDetails, pipeline.Id, repository.PRE_CD, cdStageCompleteEvent.TriggeredBy) + PreCDArtifacts, err := impl.commonArtifactService.SavePluginArtifacts(ciArtifact, cdStageCompleteEvent.PluginRegistryArtifactDetails, pipeline.Id, repository.PRE_CD, cdStageCompleteEvent.TriggeredBy) if err != nil { impl.logger.Errorw("error in saving plugin artifacts", "err", err) return err @@ -832,57 +833,6 @@ func (impl *WorkflowDagExecutorImpl) HandlePreStageSuccessEvent(triggerContext T return nil } -func (impl *WorkflowDagExecutorImpl) SavePluginArtifacts(ciArtifact *repository.CiArtifact, pluginArtifactsDetail map[string][]string, pipelineId int, stage string, triggerdBy int32) ([]*repository.CiArtifact, error) { - - saveArtifacts, err := impl.ciArtifactRepository.GetArtifactsByDataSourceAndComponentId(stage, pipelineId) - if err != nil { - return nil, err - } - PipelineArtifacts := make(map[string]bool) - for _, artifact := range saveArtifacts { - PipelineArtifacts[artifact.Image] = true - } - var parentCiArtifactId int - if ciArtifact.ParentCiArtifact > 0 { - parentCiArtifactId = ciArtifact.ParentCiArtifact - } else { - parentCiArtifactId = ciArtifact.Id - } - var CDArtifacts []*repository.CiArtifact - for registry, artifacts := range pluginArtifactsDetail { - // artifacts are list of images - for _, artifact := range artifacts { - _, artifactAlreadySaved := PipelineArtifacts[artifact] - if artifactAlreadySaved { - continue - } - pluginArtifact := &repository.CiArtifact{ - Image: artifact, - ImageDigest: ciArtifact.ImageDigest, - MaterialInfo: ciArtifact.MaterialInfo, - DataSource: stage, - ComponentId: pipelineId, - CredentialsSourceType: repository.GLOBAL_CONTAINER_REGISTRY, - CredentialSourceValue: registry, - AuditLog: sql.AuditLog{ - CreatedOn: time.Now(), - CreatedBy: triggerdBy, - UpdatedOn: time.Now(), - UpdatedBy: triggerdBy, - }, - ParentCiArtifact: parentCiArtifactId, - } - CDArtifacts = append(CDArtifacts, pluginArtifact) - } - } - _, err = impl.ciArtifactRepository.SaveAll(CDArtifacts) - if err != nil { - impl.logger.Errorw("Error in saving artifacts metadata generated by plugin") - return CDArtifacts, err - } - return CDArtifacts, nil -} - func (impl *WorkflowDagExecutorImpl) TriggerPreStage(request TriggerRequest) error { //setting triggeredAt variable to have consistent data for various audit log places in db for deployment time triggeredAt := time.Now() @@ -1856,7 +1806,7 @@ func (impl *WorkflowDagExecutorImpl) HandlePostStageSuccessEvent(triggerContext return err } if len(pluginRegistryImageDetails) > 0 { - PostCDArtifacts, err := impl.SavePluginArtifacts(ciArtifact, pluginRegistryImageDetails, cdPipelineId, repository.POST_CD, triggeredBy) + PostCDArtifacts, err := impl.commonArtifactService.SavePluginArtifacts(ciArtifact, pluginRegistryImageDetails, cdPipelineId, repository.POST_CD, triggeredBy) if err != nil { impl.logger.Errorw("error in saving plugin artifacts", "err", err) return err @@ -2488,7 +2438,7 @@ func (impl *WorkflowDagExecutorImpl) HandleCDTriggerRelease(overrideRequest *bea } // synchronous mode of installation starts - valuesOverrideResponse, builtChartPath, err := impl.BuildManifestForTrigger(overrideRequest, triggeredAt, ctx) + valuesOverrideResponse, builtChartPath, err := impl.manifestCreationService.BuildManifestForTrigger(overrideRequest, triggeredAt, ctx) _, span := otel.Tracer("orchestrator").Start(ctx, "CreateHistoriesForDeploymentTrigger") err1 := impl.deployedConfigurationHistoryService.CreateHistoriesForDeploymentTrigger(valuesOverrideResponse.Pipeline, valuesOverrideResponse.PipelineStrategy, valuesOverrideResponse.EnvOverride, triggeredAt, deployedBy) if err1 != nil { @@ -2578,21 +2528,6 @@ func (impl *WorkflowDagExecutorImpl) SetPipelineFieldsInOverrideRequest(override // write integration/unit test for each function -func (impl *WorkflowDagExecutorImpl) BuildManifestForTrigger(overrideRequest *bean.ValuesOverrideRequest, triggeredAt time.Time, ctx context.Context) (valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, err error) { - valuesOverrideResponse = &app.ValuesOverrideResponse{} - valuesOverrideResponse, err = impl.manifestCreationService.GetValuesOverrideForTrigger(overrideRequest, triggeredAt, ctx) - if err != nil { - impl.logger.Errorw("error in fetching values for trigger", "err", err) - return valuesOverrideResponse, "", err - } - builtChartPath, err = impl.deploymentTemplateService.BuildChartAndGetPath(overrideRequest.AppName, valuesOverrideResponse.EnvOverride, ctx) - if err != nil { - impl.logger.Errorw("error in parsing reference chart", "err", err) - return valuesOverrideResponse, "", err - } - return valuesOverrideResponse, builtChartPath, err -} - func (impl *WorkflowDagExecutorImpl) updatePipeline(pipeline *pipelineConfig.Pipeline, userId int32) (bool, error) { err := impl.pipelineRepository.SetDeploymentAppCreatedInPipeline(true, pipeline.Id, userId) if err != nil { From 1ead439f75067361f31a5f38e620652f8c27d172 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Fri, 2 Feb 2024 17:58:08 +0530 Subject: [PATCH 57/64] extracted artifact logic from workflowDAG --- pkg/build/artifacts/CommonArtifactService.go | 82 ++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 pkg/build/artifacts/CommonArtifactService.go diff --git a/pkg/build/artifacts/CommonArtifactService.go b/pkg/build/artifacts/CommonArtifactService.go new file mode 100644 index 0000000000..636ce434eb --- /dev/null +++ b/pkg/build/artifacts/CommonArtifactService.go @@ -0,0 +1,82 @@ +package artifacts + +import ( + "github.com/devtron-labs/devtron/internal/sql/repository" + "github.com/devtron-labs/devtron/pkg/sql" + "go.uber.org/zap" + "time" +) + +//this service is created with the thought of as a common interaction point for artifact related operations required +//for build and deployment. Can be updated/removed as per evolution patterns. +//ques can be asked - why this is not outside /build/artifact? Because the place used for storing artifacts is still known as ci_artifact and a lot of current impl +//has duplicated logic, until it is extracted and the design pattern is made easily justifiable, have placed this inside /build/artifact. + +type CommonArtifactService interface { + SavePluginArtifacts(ciArtifact *repository.CiArtifact, pluginArtifactsDetail map[string][]string, + pipelineId int, stage string, triggeredBy int32) ([]*repository.CiArtifact, error) +} + +type CommonArtifactServiceImpl struct { + logger *zap.SugaredLogger + ciArtifactRepository repository.CiArtifactRepository +} + +func NewCommonArtifactServiceImpl(logger *zap.SugaredLogger, + ciArtifactRepository repository.CiArtifactRepository) *CommonArtifactServiceImpl { + return &CommonArtifactServiceImpl{ + logger: logger, + ciArtifactRepository: ciArtifactRepository, + } +} + +func (impl *CommonArtifactServiceImpl) SavePluginArtifacts(ciArtifact *repository.CiArtifact, pluginArtifactsDetail map[string][]string, + pipelineId int, stage string, triggeredBy int32) ([]*repository.CiArtifact, error) { + saveArtifacts, err := impl.ciArtifactRepository.GetArtifactsByDataSourceAndComponentId(stage, pipelineId) + if err != nil { + return nil, err + } + PipelineArtifacts := make(map[string]bool) + for _, artifact := range saveArtifacts { + PipelineArtifacts[artifact.Image] = true + } + var parentCiArtifactId int + if ciArtifact.ParentCiArtifact > 0 { + parentCiArtifactId = ciArtifact.ParentCiArtifact + } else { + parentCiArtifactId = ciArtifact.Id + } + var CDArtifacts []*repository.CiArtifact + for registry, artifacts := range pluginArtifactsDetail { + // artifacts are list of images + for _, artifact := range artifacts { + _, artifactAlreadySaved := PipelineArtifacts[artifact] + if artifactAlreadySaved { + continue + } + pluginArtifact := &repository.CiArtifact{ + Image: artifact, + ImageDigest: ciArtifact.ImageDigest, + MaterialInfo: ciArtifact.MaterialInfo, + DataSource: stage, + ComponentId: pipelineId, + CredentialsSourceType: repository.GLOBAL_CONTAINER_REGISTRY, + CredentialSourceValue: registry, + AuditLog: sql.AuditLog{ + CreatedOn: time.Now(), + CreatedBy: triggeredBy, + UpdatedOn: time.Now(), + UpdatedBy: triggeredBy, + }, + ParentCiArtifact: parentCiArtifactId, + } + CDArtifacts = append(CDArtifacts, pluginArtifact) + } + } + _, err = impl.ciArtifactRepository.SaveAll(CDArtifacts) + if err != nil { + impl.logger.Errorw("Error in saving artifacts metadata generated by plugin") + return CDArtifacts, err + } + return CDArtifacts, nil +} From 7d1787b59310773482ecaf2546667484f30a703a Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Mon, 5 Feb 2024 16:22:43 +0530 Subject: [PATCH 58/64] refactoring --- Wire.go | 17 +- api/restHandler/ExternalCiRestHandler.go | 40 +- api/restHandler/PipelineTriggerRestHandler.go | 85 +- api/router/pubsub/ApplicationStatusHandler.go | 16 +- api/router/pubsub/CiEventHandler.go | 358 --- api/router/router.go | 4 +- .../cron/CdApplicationStatusUpdateHandler.go | 45 +- pkg/app/AppService.go | 31 +- pkg/app/DeploymentEventHandler.go | 1 + pkg/build/artifacts/wire_artifacts.go | 8 + pkg/build/wire_build.go | 10 + pkg/bulkAction/BulkUpdateService.go | 49 +- .../deployedApp/DeployedAppService.go | 82 +- pkg/deployment/deployedApp/bean/bean.go | 17 + .../devtronApps/PostStageTriggerService.go | 142 + .../devtronApps/PrePostStageTriggerService.go | 864 ++++++ .../trigger/devtronApps/TriggerService.go | 664 ++++- .../CentralEventProcessorService.go | 5 + pkg/eventProcessor/bean/workflowEventBean.go | 26 + .../in/WorkflowEventProcessorService.go | 247 +- .../out/WorkflowEventPublishService.go | 37 +- pkg/eventProcessor/out/bean/bean.go | 17 + pkg/pipeline/CdHandler.go | 699 +---- pkg/pipeline/WebhookService.go | 393 +-- pkg/pipeline/WorkflowDagExecutor.go | 2579 ----------------- pkg/workflow/cd/CdWorkflowCommonService.go | 169 +- pkg/workflow/dag/WorkflowDagExecutor.go | 1229 ++++++++ pkg/workflow/dag/bean/bean.go | 20 + pkg/workflow/status/WorkflowStatusService.go | 613 ++++ pkg/workflow/status/bean/bean.go | 8 + pkg/workflow/status/wire_workflowStatus.go | 8 + pkg/workflow/wire_workflow.go | 12 + util/GlobalConfig.go | 1 + wire_gen.go | 171 +- 34 files changed, 4417 insertions(+), 4250 deletions(-) delete mode 100644 api/router/pubsub/CiEventHandler.go create mode 100644 pkg/build/artifacts/wire_artifacts.go create mode 100644 pkg/build/wire_build.go create mode 100644 pkg/deployment/trigger/devtronApps/PostStageTriggerService.go create mode 100644 pkg/deployment/trigger/devtronApps/PrePostStageTriggerService.go delete mode 100644 pkg/pipeline/WorkflowDagExecutor.go create mode 100644 pkg/workflow/dag/WorkflowDagExecutor.go create mode 100644 pkg/workflow/dag/bean/bean.go create mode 100644 pkg/workflow/status/WorkflowStatusService.go create mode 100644 pkg/workflow/status/bean/bean.go create mode 100644 pkg/workflow/status/wire_workflowStatus.go create mode 100644 pkg/workflow/wire_workflow.go diff --git a/Wire.go b/Wire.go index 3366f1ca8d..8f762cfac8 100644 --- a/Wire.go +++ b/Wire.go @@ -93,6 +93,7 @@ import ( appStoreDeploymentGitopsTool "github.com/devtron-labs/devtron/pkg/appStore/deployment/tool" "github.com/devtron-labs/devtron/pkg/appWorkflow" "github.com/devtron-labs/devtron/pkg/attributes" + "github.com/devtron-labs/devtron/pkg/build" "github.com/devtron-labs/devtron/pkg/bulkAction" "github.com/devtron-labs/devtron/pkg/chart" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" @@ -128,7 +129,8 @@ import ( "github.com/devtron-labs/devtron/pkg/variables" "github.com/devtron-labs/devtron/pkg/variables/parsers" repository10 "github.com/devtron-labs/devtron/pkg/variables/repository" - "github.com/devtron-labs/devtron/pkg/workflow/cd" + "github.com/devtron-labs/devtron/pkg/workflow" + "github.com/devtron-labs/devtron/pkg/workflow/dag" util2 "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/argo" cron2 "github.com/devtron-labs/devtron/util/cron" @@ -163,9 +165,10 @@ func InitializeApp() (*App, error) { apiToken.ApiTokenWireSet, webhookHelm.WebhookHelmWireSet, terminal.TerminalWireSet, + build.BuildWireSet, deployment2.DeploymentWireSet, eventProcessor.EventProcessorWireSet, - cd.CdWorkflowWireSet, + workflow.WorkflowWireSet, // -------wireset end ---------- //------- gitSensor.GetConfig, @@ -454,15 +457,9 @@ func InitializeApp() (*App, error) { pubsub.NewApplicationStatusHandlerImpl, wire.Bind(new(pubsub.ApplicationStatusHandler), new(*pubsub.ApplicationStatusHandlerImpl)), - pubsub.GetCiEventConfig, - pubsub.NewCiEventHandlerImpl, - wire.Bind(new(pubsub.CiEventHandler), new(*pubsub.CiEventHandlerImpl)), - rbac.NewEnforcerUtilImpl, wire.Bind(new(rbac.EnforcerUtil), new(*rbac.EnforcerUtilImpl)), - app.NewDeploymentEventHandlerImpl, - wire.Bind(new(app.DeploymentEventHandler), new(*app.DeploymentEventHandlerImpl)), chartConfig.NewPipelineConfigRepository, wire.Bind(new(chartConfig.PipelineConfigRepository), new(*chartConfig.PipelineConfigRepositoryImpl)), @@ -545,8 +542,8 @@ func InitializeApp() (*App, error) { pipeline.NewBlobStorageConfigServiceImpl, wire.Bind(new(pipeline.BlobStorageConfigService), new(*pipeline.BlobStorageConfigServiceImpl)), - pipeline.NewWorkflowDagExecutorImpl, - wire.Bind(new(pipeline.WorkflowDagExecutor), new(*pipeline.WorkflowDagExecutorImpl)), + dag.NewWorkflowDagExecutorImpl, + wire.Bind(new(dag.WorkflowDagExecutor), new(*dag.WorkflowDagExecutorImpl)), appClone.NewAppCloneServiceImpl, wire.Bind(new(appClone.AppCloneService), new(*appClone.AppCloneServiceImpl)), diff --git a/api/restHandler/ExternalCiRestHandler.go b/api/restHandler/ExternalCiRestHandler.go index a02d2a34cd..65be56daaa 100644 --- a/api/restHandler/ExternalCiRestHandler.go +++ b/api/restHandler/ExternalCiRestHandler.go @@ -19,15 +19,14 @@ package restHandler import ( "encoding/json" + "github.com/devtron-labs/devtron/pkg/workflow/dag" "net/http" "strconv" "github.com/devtron-labs/devtron/api/restHandler/common" - "github.com/devtron-labs/devtron/api/router/pubsub" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" "github.com/devtron-labs/devtron/pkg/auth/user" "github.com/devtron-labs/devtron/pkg/pipeline" - "github.com/devtron-labs/devtron/util/rbac" "github.com/gorilla/mux" "go.uber.org/zap" "gopkg.in/go-playground/validator.v9" @@ -38,27 +37,22 @@ type ExternalCiRestHandler interface { } type ExternalCiRestHandlerImpl struct { - logger *zap.SugaredLogger - webhookService pipeline.WebhookService - ciEventHandler pubsub.CiEventHandler - validator *validator.Validate - userService user.UserService - enforcer casbin.Enforcer - enforcerUtil rbac.EnforcerUtil + logger *zap.SugaredLogger + validator *validator.Validate + userService user.UserService + enforcer casbin.Enforcer + workflowDagExecutor dag.WorkflowDagExecutor } -func NewExternalCiRestHandlerImpl(logger *zap.SugaredLogger, webhookService pipeline.WebhookService, - ciEventHandler pubsub.CiEventHandler, validator *validator.Validate, userService user.UserService, - enforcer casbin.Enforcer, - enforcerUtil rbac.EnforcerUtil) *ExternalCiRestHandlerImpl { +func NewExternalCiRestHandlerImpl(logger *zap.SugaredLogger, validator *validator.Validate, + userService user.UserService, enforcer casbin.Enforcer, + workflowDagExecutor dag.WorkflowDagExecutor) *ExternalCiRestHandlerImpl { return &ExternalCiRestHandlerImpl{ - webhookService: webhookService, - logger: logger, - ciEventHandler: ciEventHandler, - validator: validator, - userService: userService, - enforcer: enforcer, - enforcerUtil: enforcerUtil, + logger: logger, + validator: validator, + userService: userService, + enforcer: enforcer, + workflowDagExecutor: workflowDagExecutor, } } @@ -77,7 +71,7 @@ func (impl ExternalCiRestHandlerImpl) HandleExternalCiWebhook(w http.ResponseWri return } decoder := json.NewDecoder(r.Body) - var req pubsub.CiCompleteEvent + var req pipeline.ExternalCiWebhookDto err = decoder.Decode(&req) if err != nil { impl.logger.Errorw("request err, HandleExternalCiWebhook", "err", err, "payload", req) @@ -94,7 +88,7 @@ func (impl ExternalCiRestHandlerImpl) HandleExternalCiWebhook(w http.ResponseWri return } //fetching request - ciArtifactReq, err := impl.ciEventHandler.BuildCiArtifactRequestForWebhook(req) + ciArtifactReq, err := impl.workflowDagExecutor.BuildCiArtifactRequestForWebhook(req) if err != nil { impl.logger.Errorw("service err, HandleExternalCiWebhook", "err", err, "payload", req) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) @@ -107,7 +101,7 @@ func (impl ExternalCiRestHandlerImpl) HandleExternalCiWebhook(w http.ResponseWri common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - _, err = impl.webhookService.HandleExternalCiWebhook(externalCiId, ciArtifactReq, impl.checkExternalCiDeploymentAuth, token) + _, err = impl.workflowDagExecutor.HandleExternalCiWebhook(externalCiId, ciArtifactReq, impl.checkExternalCiDeploymentAuth, token) if err != nil { impl.logger.Errorw("service err, HandleExternalCiWebhook", "err", err, "payload", req) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) diff --git a/api/restHandler/PipelineTriggerRestHandler.go b/api/restHandler/PipelineTriggerRestHandler.go index a943bb4e3c..125b696b62 100644 --- a/api/restHandler/PipelineTriggerRestHandler.go +++ b/api/restHandler/PipelineTriggerRestHandler.go @@ -23,6 +23,11 @@ import ( "fmt" "github.com/devtron-labs/devtron/pkg/deployment/deployedApp" bean2 "github.com/devtron-labs/devtron/pkg/deployment/deployedApp/bean" + "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps" + bean3 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" + "github.com/devtron-labs/devtron/pkg/eventProcessor/out" + bean4 "github.com/devtron-labs/devtron/pkg/eventProcessor/out/bean" + "github.com/devtron-labs/devtron/pkg/workflow/dag" "net/http" "strconv" @@ -54,38 +59,44 @@ type PipelineTriggerRestHandler interface { } type PipelineTriggerRestHandlerImpl struct { - appService app.AppService - userAuthService user.UserService - validator *validator.Validate - enforcer casbin.Enforcer - teamService team.TeamService - logger *zap.SugaredLogger - workflowDagExecutor pipeline.WorkflowDagExecutor - enforcerUtil rbac.EnforcerUtil - deploymentGroupService deploymentGroup.DeploymentGroupService - argoUserService argo.ArgoUserService - deploymentConfigService pipeline.DeploymentConfigService - deployedAppService deployedApp.DeployedAppService + appService app.AppService + userAuthService user.UserService + validator *validator.Validate + enforcer casbin.Enforcer + teamService team.TeamService + logger *zap.SugaredLogger + workflowDagExecutor dag.WorkflowDagExecutor + enforcerUtil rbac.EnforcerUtil + deploymentGroupService deploymentGroup.DeploymentGroupService + argoUserService argo.ArgoUserService + deploymentConfigService pipeline.DeploymentConfigService + deployedAppService deployedApp.DeployedAppService + cdTriggerService devtronApps.TriggerService + workflowEventPublishService out.WorkflowEventPublishService } func NewPipelineRestHandler(appService app.AppService, userAuthService user.UserService, validator *validator.Validate, enforcer casbin.Enforcer, teamService team.TeamService, logger *zap.SugaredLogger, enforcerUtil rbac.EnforcerUtil, - workflowDagExecutor pipeline.WorkflowDagExecutor, deploymentGroupService deploymentGroup.DeploymentGroupService, + workflowDagExecutor dag.WorkflowDagExecutor, deploymentGroupService deploymentGroup.DeploymentGroupService, argoUserService argo.ArgoUserService, deploymentConfigService pipeline.DeploymentConfigService, - deployedAppService deployedApp.DeployedAppService) *PipelineTriggerRestHandlerImpl { + deployedAppService deployedApp.DeployedAppService, + cdTriggerService devtronApps.TriggerService, + workflowEventPublishService out.WorkflowEventPublishService) *PipelineTriggerRestHandlerImpl { pipelineHandler := &PipelineTriggerRestHandlerImpl{ - appService: appService, - userAuthService: userAuthService, - validator: validator, - enforcer: enforcer, - teamService: teamService, - logger: logger, - workflowDagExecutor: workflowDagExecutor, - enforcerUtil: enforcerUtil, - deploymentGroupService: deploymentGroupService, - argoUserService: argoUserService, - deploymentConfigService: deploymentConfigService, - deployedAppService: deployedAppService, + appService: appService, + userAuthService: userAuthService, + validator: validator, + enforcer: enforcer, + teamService: teamService, + logger: logger, + workflowDagExecutor: workflowDagExecutor, + enforcerUtil: enforcerUtil, + deploymentGroupService: deploymentGroupService, + argoUserService: argoUserService, + deploymentConfigService: deploymentConfigService, + deployedAppService: deployedAppService, + cdTriggerService: cdTriggerService, + workflowEventPublishService: workflowEventPublishService, } return pipelineHandler } @@ -134,10 +145,10 @@ func (handler PipelineTriggerRestHandlerImpl) OverrideConfig(w http.ResponseWrit } ctx := context.WithValue(r.Context(), "token", acdToken) _, span := otel.Tracer("orchestrator").Start(ctx, "workflowDagExecutor.ManualCdTrigger") - triggerContext := pipeline.TriggerContext{ + triggerContext := bean3.TriggerContext{ Context: ctx, } - mergeResp, err := handler.workflowDagExecutor.ManualCdTrigger(triggerContext, &overrideRequest) + mergeResp, err := handler.cdTriggerService.ManualCdTrigger(triggerContext, &overrideRequest) span.End() if err != nil { handler.logger.Errorw("request err, OverrideConfig", "err", err, "payload", overrideRequest) @@ -197,7 +208,7 @@ func (handler PipelineTriggerRestHandlerImpl) StartStopApp(w http.ResponseWriter common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) return } - var overrideRequest pipeline.StopAppRequest + var overrideRequest bean2.StopAppRequest err = decoder.Decode(&overrideRequest) if err != nil { handler.logger.Errorw("request err, StartStopApp", "err", err, "payload", overrideRequest) @@ -232,10 +243,7 @@ func (handler PipelineTriggerRestHandlerImpl) StartStopApp(w http.ResponseWriter return } ctx := context.WithValue(r.Context(), "token", acdToken) - triggerContext := pipeline.TriggerContext{ - Context: ctx, - } - mergeResp, err := handler.workflowDagExecutor.StopStartApp(triggerContext, &overrideRequest) + mergeResp, err := handler.deployedAppService.StopStartApp(ctx, &overrideRequest) if err != nil { handler.logger.Errorw("service err, StartStopApp", "err", err, "payload", overrideRequest) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) @@ -253,7 +261,7 @@ func (handler PipelineTriggerRestHandlerImpl) StartStopDeploymentGroup(w http.Re common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) return } - var stopDeploymentGroupRequest pipeline.StopDeploymentGroupRequest + var stopDeploymentGroupRequest bean4.StopDeploymentGroupRequest err = decoder.Decode(&stopDeploymentGroupRequest) if err != nil { handler.logger.Errorw("request err, StartStopDeploymentGroup", "err", err, "payload", stopDeploymentGroupRequest) @@ -289,14 +297,7 @@ func (handler PipelineTriggerRestHandlerImpl) StartStopDeploymentGroup(w http.Re return } //rback block ends here - acdToken, err := handler.argoUserService.GetLatestDevtronArgoCdUserToken() - if err != nil { - handler.logger.Errorw("error in getting acd token", "err", err) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - ctx := context.WithValue(r.Context(), "token", acdToken) - res, err := handler.workflowDagExecutor.TriggerBulkHibernateAsync(stopDeploymentGroupRequest, ctx) + res, err := handler.workflowEventPublishService.TriggerBulkHibernateAsync(stopDeploymentGroupRequest) if err != nil { handler.logger.Errorw("service err, StartStopDeploymentGroup", "err", err, "payload", stopDeploymentGroupRequest) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) diff --git a/api/router/pubsub/ApplicationStatusHandler.go b/api/router/pubsub/ApplicationStatusHandler.go index e59165e701..861e7979f9 100644 --- a/api/router/pubsub/ApplicationStatusHandler.go +++ b/api/router/pubsub/ApplicationStatusHandler.go @@ -25,6 +25,9 @@ import ( "github.com/devtron-labs/common-lib/pubsub-lib/model" "github.com/devtron-labs/devtron/pkg/app" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" + bean2 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" + "github.com/devtron-labs/devtron/pkg/workflow/cd" + "github.com/devtron-labs/devtron/pkg/workflow/dag" "k8s.io/utils/pointer" "time" @@ -51,20 +54,22 @@ type ApplicationStatusHandlerImpl struct { logger *zap.SugaredLogger pubsubClient *pubsub.PubSubClientServiceImpl appService app.AppService - workflowDagExecutor pipeline.WorkflowDagExecutor + workflowDagExecutor dag.WorkflowDagExecutor installedAppService service.InstalledAppService appStoreDeploymentService service.AppStoreDeploymentService pipelineBuilder pipeline.PipelineBuilder pipelineRepository pipelineConfig.PipelineRepository installedAppRepository repository4.InstalledAppRepository gitOpsConfigReadService config.GitOpsConfigReadService + cdWorkflowCommonService cd.CdWorkflowCommonService } func NewApplicationStatusHandlerImpl(logger *zap.SugaredLogger, pubsubClient *pubsub.PubSubClientServiceImpl, appService app.AppService, - workflowDagExecutor pipeline.WorkflowDagExecutor, installedAppService service.InstalledAppService, + workflowDagExecutor dag.WorkflowDagExecutor, installedAppService service.InstalledAppService, appStoreDeploymentService service.AppStoreDeploymentService, pipelineBuilder pipeline.PipelineBuilder, pipelineRepository pipelineConfig.PipelineRepository, installedAppRepository repository4.InstalledAppRepository, - gitOpsConfigReadService config.GitOpsConfigReadService) *ApplicationStatusHandlerImpl { + gitOpsConfigReadService config.GitOpsConfigReadService, + cdWorkflowCommonService cd.CdWorkflowCommonService) *ApplicationStatusHandlerImpl { appStatusUpdateHandlerImpl := &ApplicationStatusHandlerImpl{ logger: logger, pubsubClient: pubsubClient, @@ -76,6 +81,7 @@ func NewApplicationStatusHandlerImpl(logger *zap.SugaredLogger, pubsubClient *pu pipelineRepository: pipelineRepository, installedAppRepository: installedAppRepository, gitOpsConfigReadService: gitOpsConfigReadService, + cdWorkflowCommonService: cdWorkflowCommonService, } err := appStatusUpdateHandlerImpl.Subscribe() if err != nil { @@ -151,7 +157,7 @@ func (impl *ApplicationStatusHandlerImpl) Subscribe() error { // invoke DagExecutor, for cd success which will trigger post stage if exist. if isSucceeded { impl.logger.Debugw("git hash history", "list", app.Status.History) - triggerContext := pipeline.TriggerContext{ + triggerContext := bean2.TriggerContext{ ReferenceId: pointer.String(msg.MsgId), } err = impl.workflowDagExecutor.HandleDeploymentSuccessEvent(triggerContext, pipelineOverride) @@ -168,7 +174,7 @@ func (impl *ApplicationStatusHandlerImpl) Subscribe() error { return "", nil } - validations := impl.workflowDagExecutor.GetTriggerValidateFuncs() + validations := impl.cdWorkflowCommonService.GetTriggerValidateFuncs() err := impl.pubsubClient.Subscribe(pubsub.APPLICATION_STATUS_UPDATE_TOPIC, callback, loggerFunc, validations...) if err != nil { impl.logger.Error(err) diff --git a/api/router/pubsub/CiEventHandler.go b/api/router/pubsub/CiEventHandler.go deleted file mode 100644 index a759854dc3..0000000000 --- a/api/router/pubsub/CiEventHandler.go +++ /dev/null @@ -1,358 +0,0 @@ -/* - * Copyright (c) 2020 Devtron Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package pubsub - -import ( - "encoding/json" - "fmt" - "github.com/aws/aws-sdk-go-v2/service/ecr/types" - "github.com/caarlos0/env/v6" - pubsub "github.com/devtron-labs/common-lib/pubsub-lib" - "github.com/devtron-labs/common-lib/pubsub-lib/model" - "github.com/devtron-labs/devtron/internal/sql/repository" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/devtron-labs/devtron/pkg/pipeline" - bean2 "github.com/devtron-labs/devtron/pkg/pipeline/bean" - "github.com/devtron-labs/devtron/util" - "go.uber.org/zap" - "gopkg.in/go-playground/validator.v9" - "k8s.io/utils/pointer" - "time" -) - -type CiEventConfig struct { - ExposeCiMetrics bool `env:"EXPOSE_CI_METRICS" envDefault:"false"` -} - -func GetCiEventConfig() (*CiEventConfig, error) { - cfg := &CiEventConfig{} - err := env.Parse(cfg) - return cfg, err -} - -type CiEventHandler interface { - Subscribe() error - BuildCiArtifactRequest(event CiCompleteEvent) (*pipeline.CiArtifactWebhookRequest, error) - BuildCiArtifactRequestForWebhook(event CiCompleteEvent) (*pipeline.CiArtifactWebhookRequest, error) -} - -type CiEventHandlerImpl struct { - logger *zap.SugaredLogger - pubsubClient *pubsub.PubSubClientServiceImpl - webhookService pipeline.WebhookService - validator *validator.Validate - ciEventConfig *CiEventConfig -} - -type ImageDetailsFromCR struct { - ImageDetails []types.ImageDetail `json:"imageDetails"` - Region string `json:"region"` -} - -type CiCompleteEvent struct { - CiProjectDetails []bean2.CiProjectDetails `json:"ciProjectDetails"` - DockerImage string `json:"dockerImage" validate:"required,image-validator"` - Digest string `json:"digest"` - PipelineId int `json:"pipelineId"` - WorkflowId *int `json:"workflowId"` - TriggeredBy int32 `json:"triggeredBy"` - PipelineName string `json:"pipelineName"` - DataSource string `json:"dataSource"` - MaterialType string `json:"materialType"` - Metrics util.CIMetrics `json:"metrics"` - AppName string `json:"appName"` - IsArtifactUploaded bool `json:"isArtifactUploaded"` - FailureReason string `json:"failureReason"` - ImageDetailsFromCR *ImageDetailsFromCR `json:"imageDetailsFromCR"` - PluginRegistryArtifactDetails map[string][]string `json:"PluginRegistryArtifactDetails"` - PluginArtifactStage string `json:"pluginArtifactStage"` -} - -func NewCiEventHandlerImpl(logger *zap.SugaredLogger, pubsubClient *pubsub.PubSubClientServiceImpl, webhookService pipeline.WebhookService, validator *validator.Validate, ciEventConfig *CiEventConfig) *CiEventHandlerImpl { - ciEventHandlerImpl := &CiEventHandlerImpl{ - logger: logger, - pubsubClient: pubsubClient, - webhookService: webhookService, - validator: validator, - ciEventConfig: ciEventConfig, - } - err := ciEventHandlerImpl.Subscribe() - if err != nil { - logger.Error(err) - return nil - } - return ciEventHandlerImpl -} - -func (impl *CiEventHandlerImpl) Subscribe() error { - callback := func(msg *model.PubSubMsg) { - ciCompleteEvent := CiCompleteEvent{} - err := json.Unmarshal([]byte(string(msg.Data)), &ciCompleteEvent) - if err != nil { - impl.logger.Error("error while unmarshalling json data", "error", err) - return - } - impl.logger.Debugw("ci complete event for ci", "ciPipelineId", ciCompleteEvent.PipelineId) - req, err := impl.BuildCiArtifactRequest(ciCompleteEvent) - if err != nil { - return - } - - triggerContext := pipeline.TriggerContext{ - ReferenceId: pointer.String(msg.MsgId), - } - - if ciCompleteEvent.FailureReason != "" { - req.FailureReason = ciCompleteEvent.FailureReason - err := impl.webhookService.HandleCiStepFailedEvent(ciCompleteEvent.PipelineId, req) - if err != nil { - impl.logger.Error("Error while sending event for CI failure for pipelineID: ", - ciCompleteEvent.PipelineId, "request: ", req, "error: ", err) - return - } - } else if ciCompleteEvent.ImageDetailsFromCR != nil { - if len(ciCompleteEvent.ImageDetailsFromCR.ImageDetails) > 0 { - imageDetails := util.GetReverseSortedImageDetails(ciCompleteEvent.ImageDetailsFromCR.ImageDetails) - digestWorkflowMap, err := impl.webhookService.HandleMultipleImagesFromEvent(imageDetails, *ciCompleteEvent.WorkflowId) - if err != nil { - impl.logger.Errorw("error in getting digest workflow map", "err", err, "workflowId", ciCompleteEvent.WorkflowId) - return - } - for _, detail := range imageDetails { - request, err := impl.BuildCIArtifactRequestForImageFromCR(detail, ciCompleteEvent.ImageDetailsFromCR.Region, ciCompleteEvent, digestWorkflowMap[*detail.ImageDigest].Id) - if err != nil { - impl.logger.Error("Error while creating request for pipelineID", "pipelineId", ciCompleteEvent.PipelineId, "err", err) - return - } - resp, err := impl.ValidateAndHandleCiSuccessEvent(triggerContext, ciCompleteEvent.PipelineId, request, detail.ImagePushedAt) - if err != nil { - return - } - impl.logger.Debug("response of handle ci success event for multiple images from plugin", "resp", resp) - } - } - - } else { - util.TriggerCIMetrics(ciCompleteEvent.Metrics, impl.ciEventConfig.ExposeCiMetrics, ciCompleteEvent.PipelineName, ciCompleteEvent.AppName) - resp, err := impl.ValidateAndHandleCiSuccessEvent(triggerContext, ciCompleteEvent.PipelineId, req, &time.Time{}) - if err != nil { - return - } - impl.logger.Debug(resp) - } - } - - // add required logging here - var loggerFunc pubsub.LoggerFunc = func(msg model.PubSubMsg) (string, []interface{}) { - ciCompleteEvent := CiCompleteEvent{} - err := json.Unmarshal([]byte(string(msg.Data)), &ciCompleteEvent) - if err != nil { - return "error while unmarshalling json data", []interface{}{"error", err} - } - return "got message for ci-completion", []interface{}{"ciPipelineId", ciCompleteEvent.PipelineId, "workflowId", ciCompleteEvent.WorkflowId} - } - - validations := impl.webhookService.GetTriggerValidateFuncs() - err := impl.pubsubClient.Subscribe(pubsub.CI_COMPLETE_TOPIC, callback, loggerFunc, validations...) - if err != nil { - impl.logger.Error(err) - return err - } - return nil -} - -func (impl *CiEventHandlerImpl) ValidateAndHandleCiSuccessEvent(triggerContext pipeline.TriggerContext, ciPipelineId int, request *pipeline.CiArtifactWebhookRequest, imagePushedAt *time.Time) (int, error) { - validationErr := impl.validator.Struct(request) - if validationErr != nil { - impl.logger.Errorw("validation err, HandleCiSuccessEvent", "err", validationErr, "payload", request) - return 0, validationErr - } - buildArtifactId, err := impl.webhookService.HandleCiSuccessEvent(triggerContext, ciPipelineId, request, imagePushedAt) - if err != nil { - impl.logger.Error("Error while sending event for CI success for pipelineID", - ciPipelineId, "request", request, "error", err) - return 0, err - } - return buildArtifactId, nil -} - -func (impl *CiEventHandlerImpl) BuildCiArtifactRequest(event CiCompleteEvent) (*pipeline.CiArtifactWebhookRequest, error) { - var ciMaterialInfos []repository.CiMaterialInfo - for _, p := range event.CiProjectDetails { - var modifications []repository.Modification - - var branch string - var tag string - var webhookData repository.WebhookData - if p.SourceType == pipelineConfig.SOURCE_TYPE_BRANCH_FIXED { - branch = p.SourceValue - } else if p.SourceType == pipelineConfig.SOURCE_TYPE_WEBHOOK { - webhookData = repository.WebhookData{ - Id: p.WebhookData.Id, - EventActionType: p.WebhookData.EventActionType, - Data: p.WebhookData.Data, - } - } - - modification := repository.Modification{ - Revision: p.CommitHash, - ModifiedTime: p.CommitTime, - Author: p.Author, - Branch: branch, - Tag: tag, - WebhookData: webhookData, - Message: p.Message, - } - - modifications = append(modifications, modification) - ciMaterialInfo := repository.CiMaterialInfo{ - Material: repository.Material{ - GitConfiguration: repository.GitConfiguration{ - URL: p.GitRepository, - }, - Type: event.MaterialType, - }, - Changed: true, - Modifications: modifications, - } - ciMaterialInfos = append(ciMaterialInfos, ciMaterialInfo) - } - - materialBytes, err := json.Marshal(ciMaterialInfos) - if err != nil { - impl.logger.Errorw("cannot build ci artifact req", "err", err) - return nil, err - } - rawMaterialInfo := json.RawMessage(materialBytes) - fmt.Printf("Raw Message : %s\n", rawMaterialInfo) - - if event.TriggeredBy == 0 { - event.TriggeredBy = 1 // system triggered event - } - - request := &pipeline.CiArtifactWebhookRequest{ - Image: event.DockerImage, - ImageDigest: event.Digest, - DataSource: event.DataSource, - PipelineName: event.PipelineName, - MaterialInfo: rawMaterialInfo, - UserId: event.TriggeredBy, - WorkflowId: event.WorkflowId, - IsArtifactUploaded: event.IsArtifactUploaded, - PluginRegistryArtifactDetails: event.PluginRegistryArtifactDetails, - PluginArtifactStage: event.PluginArtifactStage, - } - // if DataSource is empty, repository.WEBHOOK is considered as default - if request.DataSource == "" { - request.DataSource = repository.WEBHOOK - } - return request, nil -} - -func (impl *CiEventHandlerImpl) BuildCIArtifactRequestForImageFromCR(imageDetails types.ImageDetail, region string, event CiCompleteEvent, workflowId int) (*pipeline.CiArtifactWebhookRequest, error) { - if event.TriggeredBy == 0 { - event.TriggeredBy = 1 // system triggered event - } - request := &pipeline.CiArtifactWebhookRequest{ - Image: util.ExtractEcrImage(*imageDetails.RegistryId, region, *imageDetails.RepositoryName, imageDetails.ImageTags[0]), - ImageDigest: *imageDetails.ImageDigest, - DataSource: event.DataSource, - PipelineName: event.PipelineName, - UserId: event.TriggeredBy, - WorkflowId: &workflowId, - IsArtifactUploaded: event.IsArtifactUploaded, - } - if request.DataSource == "" { - request.DataSource = repository.WEBHOOK - } - return request, nil -} - -func (impl *CiEventHandlerImpl) BuildCiArtifactRequestForWebhook(event CiCompleteEvent) (*pipeline.CiArtifactWebhookRequest, error) { - ciMaterialInfos := make([]repository.CiMaterialInfo, 0) - if event.MaterialType == "" { - event.MaterialType = "git" - } - for _, p := range event.CiProjectDetails { - var modifications []repository.Modification - - var branch string - var tag string - var webhookData repository.WebhookData - if p.SourceType == pipelineConfig.SOURCE_TYPE_BRANCH_FIXED { - branch = p.SourceValue - } else if p.SourceType == pipelineConfig.SOURCE_TYPE_WEBHOOK { - webhookData = repository.WebhookData{ - Id: p.WebhookData.Id, - EventActionType: p.WebhookData.EventActionType, - Data: p.WebhookData.Data, - } - } - - modification := repository.Modification{ - Revision: p.CommitHash, - ModifiedTime: p.CommitTime, - Author: p.Author, - Branch: branch, - Tag: tag, - WebhookData: webhookData, - Message: p.Message, - } - - modifications = append(modifications, modification) - ciMaterialInfo := repository.CiMaterialInfo{ - Material: repository.Material{ - GitConfiguration: repository.GitConfiguration{ - URL: p.GitRepository, - }, - Type: event.MaterialType, - }, - Changed: true, - Modifications: modifications, - } - ciMaterialInfos = append(ciMaterialInfos, ciMaterialInfo) - } - - materialBytes, err := json.Marshal(ciMaterialInfos) - if err != nil { - impl.logger.Errorw("cannot build ci artifact req", "err", err) - return nil, err - } - rawMaterialInfo := json.RawMessage(materialBytes) - fmt.Printf("Raw Message : %s\n", rawMaterialInfo) - - if event.TriggeredBy == 0 { - event.TriggeredBy = 1 // system triggered event - } - - request := &pipeline.CiArtifactWebhookRequest{ - Image: event.DockerImage, - ImageDigest: event.Digest, - DataSource: event.DataSource, - PipelineName: event.PipelineName, - MaterialInfo: rawMaterialInfo, - UserId: event.TriggeredBy, - WorkflowId: event.WorkflowId, - IsArtifactUploaded: event.IsArtifactUploaded, - } - // if DataSource is empty, repository.WEBHOOK is considered as default - if request.DataSource == "" { - request.DataSource = repository.WEBHOOK - } - return request, nil -} diff --git a/api/router/router.go b/api/router/router.go index 4c7f779bdf..03c8ae4c89 100644 --- a/api/router/router.go +++ b/api/router/router.go @@ -75,7 +75,6 @@ type MuxRouter struct { UserRouter user.UserRouter gitWebhookHandler pubsub.GitWebhookHandler appUpdateHandler pubsub.ApplicationStatusHandler - ciEventHandler pubsub.CiEventHandler ChartRefRouter ChartRefRouter ConfigMapRouter ConfigMapRouter AppStoreRouter appStore.AppStoreRouter @@ -134,7 +133,7 @@ func NewMuxRouter(logger *zap.SugaredLogger, HelmRouter PipelineTriggerRouter, P TeamRouter team.TeamRouter, gitWebhookHandler pubsub.GitWebhookHandler, appUpdateHandler pubsub.ApplicationStatusHandler, - ciEventHandler pubsub.CiEventHandler, pubsubClient *pubsub2.PubSubClientServiceImpl, UserRouter user.UserRouter, + pubsubClient *pubsub2.PubSubClientServiceImpl, UserRouter user.UserRouter, ChartRefRouter ChartRefRouter, ConfigMapRouter ConfigMapRouter, AppStoreRouter appStore.AppStoreRouter, chartRepositoryRouter chartRepo.ChartRepositoryRouter, ReleaseMetricsRouter ReleaseMetricsRouter, deploymentGroupRouter DeploymentGroupRouter, batchOperationRouter BatchOperationRouter, chartGroupRouter chartGroup.ChartGroupRouter, imageScanRouter ImageScanRouter, @@ -172,7 +171,6 @@ func NewMuxRouter(logger *zap.SugaredLogger, HelmRouter PipelineTriggerRouter, P logger: logger, gitWebhookHandler: gitWebhookHandler, appUpdateHandler: appUpdateHandler, - ciEventHandler: ciEventHandler, pubsubClient: pubsubClient, UserRouter: UserRouter, ChartRefRouter: ChartRefRouter, diff --git a/client/cron/CdApplicationStatusUpdateHandler.go b/client/cron/CdApplicationStatusUpdateHandler.go index f9d377112d..0c7b42aacc 100644 --- a/client/cron/CdApplicationStatusUpdateHandler.go +++ b/client/cron/CdApplicationStatusUpdateHandler.go @@ -14,7 +14,12 @@ import ( "github.com/devtron-labs/devtron/pkg/app" repository2 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" "github.com/devtron-labs/devtron/pkg/appStore/deployment/service" + bean2 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/workflow/cd" + "github.com/devtron-labs/devtron/pkg/workflow/dag" + "github.com/devtron-labs/devtron/pkg/workflow/status" + bean3 "github.com/devtron-labs/devtron/pkg/workflow/status/bean" "github.com/devtron-labs/devtron/util" cron2 "github.com/devtron-labs/devtron/util/cron" "github.com/robfig/cron/v3" @@ -38,9 +43,8 @@ type CdApplicationStatusUpdateHandlerImpl struct { logger *zap.SugaredLogger cron *cron.Cron appService app.AppService - workflowDagExecutor pipeline.WorkflowDagExecutor + workflowDagExecutor dag.WorkflowDagExecutor installedAppService service.InstalledAppService - CdHandler pipeline.CdHandler AppStatusConfig *app.AppServiceConfig pubsubClient *pubsub.PubSubClientServiceImpl pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository @@ -50,16 +54,20 @@ type CdApplicationStatusUpdateHandlerImpl struct { pipelineRepository pipelineConfig.PipelineRepository installedAppVersionHistoryRepository repository2.InstalledAppVersionHistoryRepository installedAppVersionRepository repository2.InstalledAppRepository + cdWorkflowCommonService cd.CdWorkflowCommonService + workflowStatusService status.WorkflowStatusService } func NewCdApplicationStatusUpdateHandlerImpl(logger *zap.SugaredLogger, appService app.AppService, - workflowDagExecutor pipeline.WorkflowDagExecutor, installedAppService service.InstalledAppService, - CdHandler pipeline.CdHandler, AppStatusConfig *app.AppServiceConfig, pubsubClient *pubsub.PubSubClientServiceImpl, + workflowDagExecutor dag.WorkflowDagExecutor, installedAppService service.InstalledAppService, + AppStatusConfig *app.AppServiceConfig, pubsubClient *pubsub.PubSubClientServiceImpl, pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository, eventClient client2.EventClient, appListingRepository repository.AppListingRepository, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, pipelineRepository pipelineConfig.PipelineRepository, installedAppVersionHistoryRepository repository2.InstalledAppVersionHistoryRepository, - installedAppVersionRepository repository2.InstalledAppRepository, cronLogger *cron2.CronLoggerImpl) *CdApplicationStatusUpdateHandlerImpl { + installedAppVersionRepository repository2.InstalledAppRepository, cronLogger *cron2.CronLoggerImpl, + cdWorkflowCommonService cd.CdWorkflowCommonService, + workflowStatusService status.WorkflowStatusService) *CdApplicationStatusUpdateHandlerImpl { cron := cron.New( cron.WithChain(cron.SkipIfStillRunning(cronLogger), cron.Recover(cronLogger))) @@ -70,7 +78,6 @@ func NewCdApplicationStatusUpdateHandlerImpl(logger *zap.SugaredLogger, appServi appService: appService, workflowDagExecutor: workflowDagExecutor, installedAppService: installedAppService, - CdHandler: CdHandler, AppStatusConfig: AppStatusConfig, pubsubClient: pubsubClient, pipelineStatusTimelineRepository: pipelineStatusTimelineRepository, @@ -80,6 +87,8 @@ func NewCdApplicationStatusUpdateHandlerImpl(logger *zap.SugaredLogger, appServi pipelineRepository: pipelineRepository, installedAppVersionHistoryRepository: installedAppVersionHistoryRepository, installedAppVersionRepository: installedAppVersionRepository, + cdWorkflowCommonService: cdWorkflowCommonService, + workflowStatusService: workflowStatusService, } err := impl.Subscribe() @@ -107,7 +116,7 @@ func NewCdApplicationStatusUpdateHandlerImpl(logger *zap.SugaredLogger, appServi func (impl *CdApplicationStatusUpdateHandlerImpl) Subscribe() error { callback := func(msg *model.PubSubMsg) { - statusUpdateEvent := pipeline.ArgoPipelineStatusSyncEvent{} + statusUpdateEvent := bean3.ArgoPipelineStatusSyncEvent{} var err error var cdPipeline *pipelineConfig.Pipeline var installedApp repository2.InstalledApps @@ -132,11 +141,11 @@ func (impl *CdApplicationStatusUpdateHandlerImpl) Subscribe() error { } } - triggerContext := pipeline.TriggerContext{ + triggerContext := bean2.TriggerContext{ ReferenceId: pointer.String(msg.MsgId), } - err, _ = impl.CdHandler.UpdatePipelineTimelineAndStatusByLiveApplicationFetch(triggerContext, cdPipeline, installedApp, statusUpdateEvent.UserId) + err, _ = impl.workflowStatusService.UpdatePipelineTimelineAndStatusByLiveApplicationFetch(triggerContext, cdPipeline, installedApp, statusUpdateEvent.UserId) if err != nil { impl.logger.Errorw("error on argo pipeline status update", "err", err, "msg", string(msg.Data)) return @@ -145,15 +154,15 @@ func (impl *CdApplicationStatusUpdateHandlerImpl) Subscribe() error { // add required logging here var loggerFunc pubsub.LoggerFunc = func(msg model.PubSubMsg) (string, []interface{}) { - statusUpdateEvent := pipeline.ArgoPipelineStatusSyncEvent{} - err := json.Unmarshal([]byte(string(msg.Data)), &statusUpdateEvent) + statusUpdateEvent := bean3.ArgoPipelineStatusSyncEvent{} + err := json.Unmarshal([]byte(msg.Data), &statusUpdateEvent) if err != nil { return "unmarshal error on argo pipeline status update event", []interface{}{"err", err} } return "got message for argo pipeline status update", []interface{}{"pipelineId", statusUpdateEvent.PipelineId, "installedAppVersionId", statusUpdateEvent.InstalledAppVersionId, "isAppStoreApplication", statusUpdateEvent.IsAppStoreApplication} } - validations := impl.workflowDagExecutor.GetTriggerValidateFuncs() + validations := impl.cdWorkflowCommonService.GetTriggerValidateFuncs() err := impl.pubsubClient.Subscribe(pubsub.ARGO_PIPELINE_STATUS_UPDATE_TOPIC, callback, loggerFunc, validations...) if err != nil { impl.logger.Errorw("error in subscribing to argo application status update topic", "err", err) @@ -173,7 +182,7 @@ func (impl *CdApplicationStatusUpdateHandlerImpl) HelmApplicationStatusUpdate() return } getPipelineDeployedWithinHours := impl.AppStatusConfig.GetPipelineDeployedWithinHours - err = impl.CdHandler.CheckHelmAppStatusPeriodicallyAndUpdateInDb(HelmPipelineStatusCheckEligibleTime, getPipelineDeployedWithinHours) + err = impl.workflowStatusService.CheckHelmAppStatusPeriodicallyAndUpdateInDb(HelmPipelineStatusCheckEligibleTime, getPipelineDeployedWithinHours) if err != nil { impl.logger.Errorw("error helm app status update - cron job", "err", err) return @@ -196,7 +205,7 @@ func (impl *CdApplicationStatusUpdateHandlerImpl) ArgoApplicationStatusUpdate() return } getPipelineDeployedWithinHours := impl.AppStatusConfig.GetPipelineDeployedWithinHours - err = impl.CdHandler.CheckArgoAppStatusPeriodicallyAndUpdateInDb(getPipelineDeployedBeforeMinutes, getPipelineDeployedWithinHours) + err = impl.workflowStatusService.CheckArgoAppStatusPeriodicallyAndUpdateInDb(getPipelineDeployedBeforeMinutes, getPipelineDeployedWithinHours) if err != nil { impl.logger.Errorw("error argo app status update - cron job", "err", err) return @@ -210,7 +219,7 @@ func (impl *CdApplicationStatusUpdateHandlerImpl) ArgoPipelineTimelineUpdate() { impl.logger.Errorw("error in converting string to int", "err", err) return } - err = impl.CdHandler.CheckArgoPipelineTimelineStatusPeriodicallyAndUpdateInDb(30, degradedTime) + err = impl.workflowStatusService.CheckArgoPipelineTimelineStatusPeriodicallyAndUpdateInDb(30, degradedTime) if err != nil { impl.logger.Errorw("error argo app status update - cron job", "err", err) return @@ -225,7 +234,7 @@ func (impl *CdApplicationStatusUpdateHandlerImpl) SyncPipelineStatusForResourceT return nil } if !util.IsTerminalStatus(cdWfr.Status) { - impl.CdHandler.CheckAndSendArgoPipelineStatusSyncEventIfNeeded(pipeline.Id, 0, 1, false) + impl.workflowStatusService.CheckAndSendArgoPipelineStatusSyncEventIfNeeded(pipeline.Id, 0, 1, false) } return nil } @@ -238,7 +247,7 @@ func (impl *CdApplicationStatusUpdateHandlerImpl) SyncPipelineStatusForAppStoreF return nil } if !util.IsTerminalStatus(installedAppVersionHistory.Status) { - impl.CdHandler.CheckAndSendArgoPipelineStatusSyncEventIfNeeded(0, installedAppVersion.Id, 1, true) + impl.workflowStatusService.CheckAndSendArgoPipelineStatusSyncEventIfNeeded(0, installedAppVersion.Id, 1, true) } return nil } @@ -267,7 +276,7 @@ func (impl *CdApplicationStatusUpdateHandlerImpl) ManualSyncPipelineStatus(appId cdPipeline = cdPipelines[0] } - err, isTimelineUpdated := impl.CdHandler.UpdatePipelineTimelineAndStatusByLiveApplicationFetch(pipeline.TriggerContext{}, cdPipeline, installedApp, userId) + err, isTimelineUpdated := impl.workflowStatusService.UpdatePipelineTimelineAndStatusByLiveApplicationFetch(bean2.TriggerContext{}, cdPipeline, installedApp, userId) if err != nil { impl.logger.Errorw("error on argo pipeline status update", "err", err) return nil diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index e2816cf2d4..13ad78f08b 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -25,7 +25,6 @@ import ( "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate" - "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" bean4 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "io/ioutil" "net/url" @@ -112,10 +111,8 @@ type AppServiceImpl struct { appStatusService appStatus.AppStatusService installedAppRepository repository4.InstalledAppRepository installedAppVersionHistoryRepository repository4.InstalledAppVersionHistoryRepository - globalEnvVariables *util2.GlobalEnvVariables scopedVariableManager variables.ScopedVariableCMCSManager acdConfig *argocdServer.ACDConfig - chartRefService chartRef.ChartRefService gitOpsConfigReadService config.GitOpsConfigReadService gitOperationService git.GitOperationService deploymentTemplateService deploymentTemplate.DeploymentTemplateService @@ -145,31 +142,24 @@ type AppService interface { func NewAppService( environmentConfigRepository chartConfig.EnvConfigOverrideRepository, pipelineOverrideRepository chartConfig.PipelineOverrideRepository, - mergeUtil *MergeUtil, - logger *zap.SugaredLogger, + mergeUtil *MergeUtil, logger *zap.SugaredLogger, pipelineRepository pipelineConfig.PipelineRepository, - eventClient client.EventClient, - eventFactory client.EventFactory, acdClient application.ServiceClient, - appRepository app.AppRepository, + eventClient client.EventClient, eventFactory client.EventFactory, + acdClient application.ServiceClient, appRepository app.AppRepository, configMapRepository chartConfig.ConfigMapRepository, chartRepository chartRepoRepository.ChartRepository, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, commonService commonService.CommonService, - chartTemplateService ChartTemplateService, - argoUserService argo.ArgoUserService, + chartTemplateService ChartTemplateService, argoUserService argo.ArgoUserService, cdPipelineStatusTimelineRepo pipelineConfig.PipelineStatusTimelineRepository, pipelineStatusTimelineResourcesService status2.PipelineStatusTimelineResourcesService, pipelineStatusSyncDetailService status2.PipelineStatusSyncDetailService, pipelineStatusTimelineService status2.PipelineStatusTimelineService, - appStatusConfig *AppServiceConfig, - appStatusService appStatus.AppStatusService, + appStatusConfig *AppServiceConfig, appStatusService appStatus.AppStatusService, installedAppRepository repository4.InstalledAppRepository, installedAppVersionHistoryRepository repository4.InstalledAppVersionHistoryRepository, - globalEnvVariables *util2.GlobalEnvVariables, - scopedVariableManager variables.ScopedVariableCMCSManager, - acdConfig *argocdServer.ACDConfig, chartRefService chartRef.ChartRefService, - gitOpsConfigReadService config.GitOpsConfigReadService, - gitOperationService git.GitOperationService, + scopedVariableManager variables.ScopedVariableCMCSManager, acdConfig *argocdServer.ACDConfig, + gitOpsConfigReadService config.GitOpsConfigReadService, gitOperationService git.GitOperationService, deploymentTemplateService deploymentTemplate.DeploymentTemplateService) *AppServiceImpl { appServiceImpl := &AppServiceImpl{ environmentConfigRepository: environmentConfigRepository, @@ -195,10 +185,8 @@ func NewAppService( appStatusService: appStatusService, installedAppRepository: installedAppRepository, installedAppVersionHistoryRepository: installedAppVersionHistoryRepository, - globalEnvVariables: globalEnvVariables, scopedVariableManager: scopedVariableManager, acdConfig: acdConfig, - chartRefService: chartRefService, gitOpsConfigReadService: gitOpsConfigReadService, gitOperationService: gitOperationService, deploymentTemplateService: deploymentTemplateService, @@ -206,11 +194,6 @@ func NewAppService( return appServiceImpl } -const ( - Success = "SUCCESS" - Failure = "FAILURE" -) - func (impl *AppServiceImpl) UpdateReleaseStatus(updateStatusRequest *bean.ReleaseStatusUpdateRequest) (bool, error) { count, err := impl.pipelineOverrideRepository.UpdateStatusByRequestIdentifier(updateStatusRequest.RequestId, updateStatusRequest.NewStatus) if err != nil { diff --git a/pkg/app/DeploymentEventHandler.go b/pkg/app/DeploymentEventHandler.go index adbaa688fd..c155af5340 100644 --- a/pkg/app/DeploymentEventHandler.go +++ b/pkg/app/DeploymentEventHandler.go @@ -29,6 +29,7 @@ import ( "go.uber.org/zap" ) +// DeploymentEventHandler is not being used, TODO: find reason for creation and then remove type DeploymentEventHandler interface { WriteCDDeploymentEvent(pipelineId, appId, envId int, eventType util.EventType) } diff --git a/pkg/build/artifacts/wire_artifacts.go b/pkg/build/artifacts/wire_artifacts.go new file mode 100644 index 0000000000..f740662298 --- /dev/null +++ b/pkg/build/artifacts/wire_artifacts.go @@ -0,0 +1,8 @@ +package artifacts + +import "github.com/google/wire" + +var ArtifactsWireSet = wire.NewSet( + NewCommonArtifactServiceImpl, + wire.Bind(new(CommonArtifactService), new(*CommonArtifactServiceImpl)), +) diff --git a/pkg/build/wire_build.go b/pkg/build/wire_build.go new file mode 100644 index 0000000000..de0d3c971c --- /dev/null +++ b/pkg/build/wire_build.go @@ -0,0 +1,10 @@ +package build + +import ( + "github.com/devtron-labs/devtron/pkg/build/artifacts" + "github.com/google/wire" +) + +var BuildWireSet = wire.NewSet( + artifacts.ArtifactsWireSet, +) diff --git a/pkg/bulkAction/BulkUpdateService.go b/pkg/bulkAction/BulkUpdateService.go index b4fba18f20..23a36a1b4a 100644 --- a/pkg/bulkAction/BulkUpdateService.go +++ b/pkg/bulkAction/BulkUpdateService.go @@ -23,15 +23,20 @@ import ( bean2 "github.com/devtron-labs/devtron/pkg/bean" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/deployment/deployedApp" + bean5 "github.com/devtron-labs/devtron/pkg/deployment/deployedApp/bean" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" bean3 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/bean" + "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps" + bean4 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/devtron-labs/devtron/pkg/pipeline" - pipeline1 "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/history" repository4 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" "github.com/devtron-labs/devtron/pkg/variables" repository5 "github.com/devtron-labs/devtron/pkg/variables/repository" + "github.com/devtron-labs/devtron/pkg/workflow/cd" + "github.com/devtron-labs/devtron/pkg/workflow/dag" "github.com/devtron-labs/devtron/util/argo" "github.com/devtron-labs/devtron/util/rbac" jsonpatch "github.com/evanphx/json-patch" @@ -65,14 +70,13 @@ type BulkUpdateService interface { type BulkUpdateServiceImpl struct { bulkUpdateRepository bulkUpdate.BulkUpdateRepository - chartRepository chartRepoRepository.ChartRepository logger *zap.SugaredLogger environmentRepository repository2.EnvironmentRepository pipelineRepository pipelineConfig.PipelineRepository appRepository app.AppRepository deploymentTemplateHistoryService history.DeploymentTemplateHistoryService configMapHistoryService history.ConfigMapHistoryService - workflowDagExecutor pipeline.WorkflowDagExecutor + workflowDagExecutor dag.WorkflowDagExecutor pipelineBuilder pipeline.PipelineBuilder enforcerUtil rbac.EnforcerUtil ciHandler pipeline.CiHandler @@ -84,16 +88,18 @@ type BulkUpdateServiceImpl struct { scopedVariableManager variables.ScopedVariableManager deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService chartRefService chartRef.ChartRefService + cdTriggerService devtronApps.TriggerService + deployedAppService deployedApp.DeployedAppService + cdWorkflowCommonService cd.CdWorkflowCommonService } func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateRepository, - chartRepository chartRepoRepository.ChartRepository, logger *zap.SugaredLogger, environmentRepository repository2.EnvironmentRepository, pipelineRepository pipelineConfig.PipelineRepository, appRepository app.AppRepository, deploymentTemplateHistoryService history.DeploymentTemplateHistoryService, - configMapHistoryService history.ConfigMapHistoryService, workflowDagExecutor pipeline.WorkflowDagExecutor, + configMapHistoryService history.ConfigMapHistoryService, workflowDagExecutor dag.WorkflowDagExecutor, pipelineBuilder pipeline.PipelineBuilder, enforcerUtil rbac.EnforcerUtil, ciHandler pipeline.CiHandler, @@ -104,10 +110,12 @@ func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateReposito argoUserService argo.ArgoUserService, scopedVariableManager variables.ScopedVariableManager, deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService, - chartRefService chartRef.ChartRefService) (*BulkUpdateServiceImpl, error) { + chartRefService chartRef.ChartRefService, + cdTriggerService devtronApps.TriggerService, + deployedAppService deployedApp.DeployedAppService, + cdWorkflowCommonService cd.CdWorkflowCommonService) (*BulkUpdateServiceImpl, error) { impl := &BulkUpdateServiceImpl{ bulkUpdateRepository: bulkUpdateRepository, - chartRepository: chartRepository, logger: logger, environmentRepository: environmentRepository, pipelineRepository: pipelineRepository, @@ -126,6 +134,9 @@ func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateReposito scopedVariableManager: scopedVariableManager, deployedAppMetricsService: deployedAppMetricsService, chartRefService: chartRefService, + cdTriggerService: cdTriggerService, + deployedAppService: deployedAppService, + cdWorkflowCommonService: cdWorkflowCommonService, } err := impl.SubscribeToCdBulkTriggerTopic() @@ -1016,16 +1027,13 @@ func (impl BulkUpdateServiceImpl) BulkHibernate(request *BulkApplicationForEnvir } var hibernateReqError error //if pipeline.DeploymentAppType == util.PIPELINE_DEPLOYMENT_TYPE_ACD { - stopRequest := &pipeline1.StopAppRequest{ + stopRequest := &bean5.StopAppRequest{ AppId: pipeline.AppId, EnvironmentId: pipeline.EnvironmentId, UserId: request.UserId, - RequestType: pipeline1.STOP, + RequestType: bean5.STOP, } - triggerContext := pipeline1.TriggerContext{ - Context: ctx, - } - _, hibernateReqError = impl.workflowDagExecutor.StopStartApp(triggerContext, stopRequest) + _, hibernateReqError = impl.deployedAppService.StopStartApp(ctx, stopRequest) if hibernateReqError != nil { impl.logger.Errorw("error in hibernating application", "err", hibernateReqError, "pipeline", pipeline) pipelineResponse := response[appKey] @@ -1175,16 +1183,13 @@ func (impl BulkUpdateServiceImpl) BulkUnHibernate(request *BulkApplicationForEnv } var hibernateReqError error //if pipeline.DeploymentAppType == util.PIPELINE_DEPLOYMENT_TYPE_ACD { - stopRequest := &pipeline1.StopAppRequest{ + stopRequest := &bean5.StopAppRequest{ AppId: pipeline.AppId, EnvironmentId: pipeline.EnvironmentId, UserId: request.UserId, - RequestType: pipeline1.START, - } - triggerContext := pipeline1.TriggerContext{ - Context: ctx, + RequestType: bean5.START, } - _, hibernateReqError = impl.workflowDagExecutor.StopStartApp(triggerContext, stopRequest) + _, hibernateReqError = impl.deployedAppService.StopStartApp(ctx, stopRequest) if hibernateReqError != nil { impl.logger.Errorw("error in un-hibernating application", "err", hibernateReqError, "pipeline", pipeline) pipelineResponse := response[appKey] @@ -1402,12 +1407,12 @@ func (impl BulkUpdateServiceImpl) SubscribeToCdBulkTriggerTopic() error { return } - triggerContext := pipeline1.TriggerContext{ + triggerContext := bean4.TriggerContext{ ReferenceId: pointer.String(msg.MsgId), Context: ctx, } - _, err = impl.workflowDagExecutor.ManualCdTrigger(triggerContext, event.ValuesOverrideRequest) + _, err = impl.cdTriggerService.ManualCdTrigger(triggerContext, event.ValuesOverrideRequest) if err != nil { impl.logger.Errorw("Error triggering CD", "topic", pubsub.CD_BULK_DEPLOY_TRIGGER_TOPIC, @@ -1426,7 +1431,7 @@ func (impl BulkUpdateServiceImpl) SubscribeToCdBulkTriggerTopic() error { return "got message for trigger cd in bulk", []interface{}{"pipelineId", event.ValuesOverrideRequest.PipelineId, "appId", event.ValuesOverrideRequest.AppId, "cdWorkflowType", event.ValuesOverrideRequest.CdWorkflowType, "ciArtifactId", event.ValuesOverrideRequest.CiArtifactId} } - validations := impl.workflowDagExecutor.GetTriggerValidateFuncs() + validations := impl.cdWorkflowCommonService.GetTriggerValidateFuncs() err := impl.pubsubClient.Subscribe(pubsub.CD_BULK_DEPLOY_TRIGGER_TOPIC, callback, loggerFunc, validations...) if err != nil { impl.logger.Error("failed to subscribe to NATS topic", diff --git a/pkg/deployment/deployedApp/DeployedAppService.go b/pkg/deployment/deployedApp/DeployedAppService.go index e4f0549dd9..2020451be2 100644 --- a/pkg/deployment/deployedApp/DeployedAppService.go +++ b/pkg/deployment/deployedApp/DeployedAppService.go @@ -2,33 +2,101 @@ package deployedApp import ( "context" + "encoding/json" + "fmt" util5 "github.com/devtron-labs/common-lib/utils/k8s" + bean2 "github.com/devtron-labs/devtron/api/bean" + "github.com/devtron-labs/devtron/internal/sql/models" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/deployment/deployedApp/bean" + "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps" + bean3 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/devtron-labs/devtron/pkg/k8s" "go.uber.org/zap" ) type DeployedAppService interface { + StopStartApp(ctx context.Context, stopRequest *bean.StopAppRequest) (int, error) RotatePods(ctx context.Context, podRotateRequest *bean.PodRotateRequest) (*k8s.RotatePodResponse, error) } type DeployedAppServiceImpl struct { - logger *zap.SugaredLogger - k8sCommonService k8s.K8sCommonService - envRepository repository.EnvironmentRepository + logger *zap.SugaredLogger + k8sCommonService k8s.K8sCommonService + cdTriggerService devtronApps.TriggerService + envRepository repository.EnvironmentRepository + pipelineRepository pipelineConfig.PipelineRepository + cdWorkflowRepository pipelineConfig.CdWorkflowRepository } func NewDeployedAppServiceImpl(logger *zap.SugaredLogger, k8sCommonService k8s.K8sCommonService, - envRepository repository.EnvironmentRepository) *DeployedAppServiceImpl { + cdTriggerService devtronApps.TriggerService, + envRepository repository.EnvironmentRepository, + pipelineRepository pipelineConfig.PipelineRepository, + cdWorkflowRepository pipelineConfig.CdWorkflowRepository) *DeployedAppServiceImpl { return &DeployedAppServiceImpl{ - logger: logger, - k8sCommonService: k8sCommonService, - envRepository: envRepository, + logger: logger, + k8sCommonService: k8sCommonService, + cdTriggerService: cdTriggerService, + envRepository: envRepository, + pipelineRepository: pipelineRepository, + cdWorkflowRepository: cdWorkflowRepository, } } +func (impl *DeployedAppServiceImpl) StopStartApp(ctx context.Context, stopRequest *bean.StopAppRequest) (int, error) { + pipelines, err := impl.pipelineRepository.FindActiveByAppIdAndEnvironmentId(stopRequest.AppId, stopRequest.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in fetching pipeline", "app", stopRequest.AppId, "env", stopRequest.EnvironmentId, "err", err) + return 0, err + } + if len(pipelines) == 0 { + return 0, fmt.Errorf("no pipeline found") + } + pipeline := pipelines[0] + + //find pipeline with default + var pipelineIds []int + for _, p := range pipelines { + impl.logger.Debugw("adding pipelineId", "pipelineId", p.Id) + pipelineIds = append(pipelineIds, p.Id) + //FIXME + } + wf, err := impl.cdWorkflowRepository.FindLatestCdWorkflowByPipelineId(pipelineIds) + if err != nil { + impl.logger.Errorw("error in fetching latest release", "err", err) + return 0, err + } + stopTemplate := `{"replicaCount":0,"autoscaling":{"MinReplicas":0,"MaxReplicas":0 ,"enabled": false} }` + overrideRequest := &bean2.ValuesOverrideRequest{ + PipelineId: pipeline.Id, + AppId: stopRequest.AppId, + CiArtifactId: wf.CiArtifactId, + UserId: stopRequest.UserId, + CdWorkflowType: bean2.CD_WORKFLOW_TYPE_DEPLOY, + } + if stopRequest.RequestType == bean.STOP { + overrideRequest.AdditionalOverride = json.RawMessage([]byte(stopTemplate)) + overrideRequest.DeploymentType = models.DEPLOYMENTTYPE_STOP + } else if stopRequest.RequestType == bean.START { + overrideRequest.DeploymentType = models.DEPLOYMENTTYPE_START + } else { + return 0, fmt.Errorf("unsupported operation %s", stopRequest.RequestType) + } + triggerContext := bean3.TriggerContext{ + Context: ctx, + ReferenceId: stopRequest.ReferenceId, + } + id, err := impl.cdTriggerService.ManualCdTrigger(triggerContext, overrideRequest) + if err != nil { + impl.logger.Errorw("error in stopping app", "err", err, "appId", stopRequest.AppId, "envId", stopRequest.EnvironmentId) + return 0, err + } + return id, err +} + func (impl *DeployedAppServiceImpl) RotatePods(ctx context.Context, podRotateRequest *bean.PodRotateRequest) (*k8s.RotatePodResponse, error) { impl.logger.Infow("rotate pod request", "payload", podRotateRequest) //extract cluster id and namespace from env id diff --git a/pkg/deployment/deployedApp/bean/bean.go b/pkg/deployment/deployedApp/bean/bean.go index ba1865b3c1..6651cc53e8 100644 --- a/pkg/deployment/deployedApp/bean/bean.go +++ b/pkg/deployment/deployedApp/bean/bean.go @@ -8,3 +8,20 @@ type PodRotateRequest struct { ResourceIdentifiers []util5.ResourceIdentifier `json:"resources" validate:"required"` UserId int32 `json:"-"` } + +type RequestType string + +const ( + START RequestType = "START" + STOP RequestType = "STOP" +) + +type StopAppRequest struct { + AppId int `json:"appId" validate:"required"` + EnvironmentId int `json:"environmentId" validate:"required"` + UserId int32 `json:"userId"` + RequestType RequestType `json:"requestType" validate:"oneof=START STOP"` + // ReferenceId is a unique identifier for the workflow runner + // refer pipelineConfig.CdWorkflowRunner + ReferenceId *string +} diff --git a/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go b/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go new file mode 100644 index 0000000000..06e65a8fc8 --- /dev/null +++ b/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go @@ -0,0 +1,142 @@ +package devtronApps + +import ( + "context" + "fmt" + bean2 "github.com/devtron-labs/devtron/api/bean" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" + bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + repository3 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" + "github.com/devtron-labs/devtron/pkg/pipeline/types" + "github.com/devtron-labs/devtron/pkg/sql" + util2 "github.com/devtron-labs/devtron/util/event" + "strconv" + "time" +) + +func (impl *TriggerServiceImpl) TriggerPostStage(request bean.TriggerRequest) error { + //setting triggeredAt variable to have consistent data for various audit log places in db for deployment time + triggeredAt := time.Now() + triggeredBy := request.TriggeredBy + pipeline := request.Pipeline + cdWf := request.CdWf + + runner := &pipelineConfig.CdWorkflowRunner{ + Name: pipeline.Name, + WorkflowType: bean2.CD_WORKFLOW_TYPE_POST, + ExecutorType: impl.config.GetWorkflowExecutorType(), + Status: pipelineConfig.WorkflowStarting, // starting PostStage + TriggeredBy: triggeredBy, + StartedOn: triggeredAt, + Namespace: impl.config.GetDefaultNamespace(), + BlobStorageEnabled: impl.config.BlobStorageEnabled, + CdWorkflowId: cdWf.Id, + LogLocation: fmt.Sprintf("%s/%s%s-%s/main.log", impl.config.GetDefaultBuildLogsKeyPrefix(), strconv.Itoa(cdWf.Id), string(bean2.CD_WORKFLOW_TYPE_POST), pipeline.Name), + AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: triggeredBy, UpdatedOn: triggeredAt, UpdatedBy: triggeredBy}, + RefCdWorkflowRunnerId: request.RefCdWorkflowRunnerId, + ReferenceId: request.TriggerContext.ReferenceId, + } + var env *repository2.Environment + var err error + if pipeline.RunPostStageInEnv { + env, err = impl.envRepository.FindById(pipeline.EnvironmentId) + if err != nil { + impl.logger.Errorw(" unable to find env ", "err", err) + return err + } + runner.Namespace = env.Namespace + } + + _, err = impl.cdWorkflowRepository.SaveWorkFlowRunner(runner) + if err != nil { + return err + } + + if cdWf.CiArtifact == nil || cdWf.CiArtifact.Id == 0 { + cdWf.CiArtifact, err = impl.ciArtifactRepository.Get(cdWf.CiArtifactId) + if err != nil { + impl.logger.Errorw("error fetching artifact data", "err", err) + return err + } + } + // Migration of deprecated DataSource Type + if cdWf.CiArtifact.IsMigrationRequired() { + migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(cdWf.CiArtifact.Id) + if migrationErr != nil { + impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", cdWf.CiArtifact.Id) + } + } + //checking vulnerability for the selected image + isVulnerable, err := impl.GetArtifactVulnerabilityStatus(cdWf.CiArtifact, pipeline, context.Background()) + if err != nil { + impl.logger.Errorw("error in getting Artifact vulnerability status, TriggerPostStage", "err", err) + return err + } + if isVulnerable { + // if image vulnerable, update timeline status and return + runner.Status = pipelineConfig.WorkflowFailed + runner.Message = pipelineConfig.FOUND_VULNERABILITY + runner.FinishedOn = time.Now() + runner.UpdatedOn = time.Now() + runner.UpdatedBy = triggeredBy + err = impl.cdWorkflowRepository.UpdateWorkFlowRunner(runner) + if err != nil { + impl.logger.Errorw("error in updating wfr status due to vulnerable image", "err", err) + return err + } + return fmt.Errorf("found vulnerability for image digest %s", cdWf.CiArtifact.ImageDigest) + } + + cdStageWorkflowRequest, err := impl.buildWFRequest(runner, cdWf, pipeline, triggeredBy) + if err != nil { + impl.logger.Errorw("error in building wfRequest", "err", err, "runner", runner, "cdWf", cdWf, "pipeline", pipeline) + return err + } + cdStageWorkflowRequest.StageType = types.POST + cdStageWorkflowRequest.Pipeline = pipeline + cdStageWorkflowRequest.Env = env + cdStageWorkflowRequest.Type = bean3.CD_WORKFLOW_PIPELINE_TYPE + // handling plugin specific logic + + pluginImagePathReservationIds, err := impl.SetCopyContainerImagePluginDataInWorkflowRequest(cdStageWorkflowRequest, pipeline.Id, types.POST, cdWf.CiArtifact) + if err != nil { + runner.Status = pipelineConfig.WorkflowFailed + runner.Message = err.Error() + _ = impl.cdWorkflowRepository.UpdateWorkFlowRunner(runner) + return err + } + + _, err = impl.cdWorkflowService.SubmitWorkflow(cdStageWorkflowRequest) + if err != nil { + impl.logger.Errorw("error in submitting workflow", "err", err, "cdStageWorkflowRequest", cdStageWorkflowRequest, "pipeline", pipeline, "env", env) + return err + } + + wfr, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(context.Background(), cdWf.Id, bean2.CD_WORKFLOW_TYPE_POST) + if err != nil { + impl.logger.Errorw("error in getting wfr by workflowId and runnerType", "err", err, "wfId", cdWf.Id) + return err + } + wfr.ImagePathReservationIds = pluginImagePathReservationIds + err = impl.cdWorkflowRepository.UpdateWorkFlowRunner(&wfr) + if err != nil { + impl.logger.Error("error in updating image path reservation ids in cd workflow runner", "err", "err") + } + + event := impl.eventFactory.Build(util2.Trigger, &pipeline.Id, pipeline.AppId, &pipeline.EnvironmentId, util2.CD) + impl.logger.Debugw("event Cd Post Trigger", "event", event) + event = impl.eventFactory.BuildExtraCDData(event, &wfr, 0, bean2.CD_WORKFLOW_TYPE_POST) + _, evtErr := impl.eventClient.WriteNotificationEvent(event) + if evtErr != nil { + impl.logger.Errorw("CD trigger event not sent", "error", evtErr) + } + //creating cd config history entry + err = impl.prePostCdScriptHistoryService.CreatePrePostCdScriptHistory(pipeline, nil, repository3.POST_CD_TYPE, true, triggeredBy, triggeredAt) + if err != nil { + impl.logger.Errorw("error in creating post cd script entry", "err", err, "pipeline", pipeline) + return err + } + return nil +} diff --git a/pkg/deployment/trigger/devtronApps/PrePostStageTriggerService.go b/pkg/deployment/trigger/devtronApps/PrePostStageTriggerService.go new file mode 100644 index 0000000000..09087406d1 --- /dev/null +++ b/pkg/deployment/trigger/devtronApps/PrePostStageTriggerService.go @@ -0,0 +1,864 @@ +package devtronApps + +import ( + "context" + "encoding/json" + "fmt" + blob_storage "github.com/devtron-labs/common-lib/blob-storage" + bean2 "github.com/devtron-labs/devtron/api/bean" + gitSensorClient "github.com/devtron-labs/devtron/client/gitSensor" + "github.com/devtron-labs/devtron/internal/sql/repository" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/sql/repository/security" + "github.com/devtron-labs/devtron/internal/util" + bean4 "github.com/devtron-labs/devtron/pkg/bean" + repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" + "github.com/devtron-labs/devtron/pkg/imageDigestPolicy" + "github.com/devtron-labs/devtron/pkg/pipeline" + bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + repository3 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" + "github.com/devtron-labs/devtron/pkg/pipeline/types" + "github.com/devtron-labs/devtron/pkg/plugin" + "github.com/devtron-labs/devtron/pkg/resourceQualifiers" + "github.com/devtron-labs/devtron/pkg/sql" + util3 "github.com/devtron-labs/devtron/pkg/util" + repository5 "github.com/devtron-labs/devtron/pkg/variables/repository" + util4 "github.com/devtron-labs/devtron/util" + util2 "github.com/devtron-labs/devtron/util/event" + "github.com/go-pg/pg" + "go.opentelemetry.io/otel" + "strconv" + "strings" + "time" +) + +const ( + GIT_COMMIT_HASH_PREFIX = "GIT_COMMIT_HASH" + GIT_SOURCE_TYPE_PREFIX = "GIT_SOURCE_TYPE" + GIT_SOURCE_VALUE_PREFIX = "GIT_SOURCE_VALUE" + GIT_SOURCE_COUNT = "GIT_SOURCE_COUNT" + APP_LABEL_KEY_PREFIX = "APP_LABEL_KEY" + APP_LABEL_VALUE_PREFIX = "APP_LABEL_VALUE" + APP_LABEL_COUNT = "APP_LABEL_COUNT" + CHILD_CD_ENV_NAME_PREFIX = "CHILD_CD_ENV_NAME" + CHILD_CD_CLUSTER_NAME_PREFIX = "CHILD_CD_CLUSTER_NAME" + CHILD_CD_COUNT = "CHILD_CD_COUNT" +) + +func (impl *TriggerServiceImpl) TriggerPreStage(request bean.TriggerRequest) error { + //setting triggeredAt variable to have consistent data for various audit log places in db for deployment time + triggeredAt := time.Now() + triggeredBy := request.TriggeredBy + artifact := request.Artifact + pipeline := request.Pipeline + ctx := request.TriggerContext.Context + //in case of pre stage manual trigger auth is already applied and for auto triggers there is no need for auth check here + cdWf := request.CdWf + var err error + if cdWf == nil { + cdWf = &pipelineConfig.CdWorkflow{ + CiArtifactId: artifact.Id, + PipelineId: pipeline.Id, + AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: 1, UpdatedOn: triggeredAt, UpdatedBy: 1}, + } + err = impl.cdWorkflowRepository.SaveWorkFlow(ctx, cdWf) + if err != nil { + return err + } + } + cdWorkflowExecutorType := impl.config.GetWorkflowExecutorType() + runner := &pipelineConfig.CdWorkflowRunner{ + Name: pipeline.Name, + WorkflowType: bean2.CD_WORKFLOW_TYPE_PRE, + ExecutorType: cdWorkflowExecutorType, + Status: pipelineConfig.WorkflowStarting, // starting PreStage + TriggeredBy: triggeredBy, + StartedOn: triggeredAt, + Namespace: impl.config.GetDefaultNamespace(), + BlobStorageEnabled: impl.config.BlobStorageEnabled, + CdWorkflowId: cdWf.Id, + LogLocation: fmt.Sprintf("%s/%s%s-%s/main.log", impl.config.GetDefaultBuildLogsKeyPrefix(), strconv.Itoa(cdWf.Id), string(bean2.CD_WORKFLOW_TYPE_PRE), pipeline.Name), + AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: 1, UpdatedOn: triggeredAt, UpdatedBy: 1}, + RefCdWorkflowRunnerId: request.RefCdWorkflowRunnerId, + ReferenceId: request.TriggerContext.ReferenceId, + } + var env *repository2.Environment + if pipeline.RunPreStageInEnv { + _, span := otel.Tracer("orchestrator").Start(ctx, "envRepository.FindById") + env, err = impl.envRepository.FindById(pipeline.EnvironmentId) + span.End() + if err != nil { + impl.logger.Errorw(" unable to find env ", "err", err) + return err + } + impl.logger.Debugw("env", "env", env) + runner.Namespace = env.Namespace + } + _, span := otel.Tracer("orchestrator").Start(ctx, "cdWorkflowRepository.SaveWorkFlowRunner") + _, err = impl.cdWorkflowRepository.SaveWorkFlowRunner(runner) + span.End() + if err != nil { + return err + } + + //checking vulnerability for the selected image + isVulnerable, err := impl.GetArtifactVulnerabilityStatus(artifact, pipeline, ctx) + if err != nil { + impl.logger.Errorw("error in getting Artifact vulnerability status, TriggerPreStage", "err", err) + return err + } + if isVulnerable { + // if image vulnerable, update timeline status and return + runner.Status = pipelineConfig.WorkflowFailed + runner.Message = pipelineConfig.FOUND_VULNERABILITY + runner.FinishedOn = time.Now() + runner.UpdatedOn = time.Now() + runner.UpdatedBy = triggeredBy + err = impl.cdWorkflowRepository.UpdateWorkFlowRunner(runner) + if err != nil { + impl.logger.Errorw("error in updating wfr status due to vulnerable image", "err", err) + return err + } + return fmt.Errorf("found vulnerability for image digest %s", artifact.ImageDigest) + } + + _, span = otel.Tracer("orchestrator").Start(ctx, "buildWFRequest") + cdStageWorkflowRequest, err := impl.buildWFRequest(runner, cdWf, pipeline, triggeredBy) + span.End() + if err != nil { + return err + } + cdStageWorkflowRequest.StageType = types.PRE + // handling copyContainerImage plugin specific logic + imagePathReservationIds, err := impl.SetCopyContainerImagePluginDataInWorkflowRequest(cdStageWorkflowRequest, pipeline.Id, types.PRE, artifact) + if err != nil { + runner.Status = pipelineConfig.WorkflowFailed + runner.Message = err.Error() + _ = impl.cdWorkflowRepository.UpdateWorkFlowRunner(runner) + return err + } else { + runner.ImagePathReservationIds = imagePathReservationIds + _ = impl.cdWorkflowRepository.UpdateWorkFlowRunner(runner) + } + + _, span = otel.Tracer("orchestrator").Start(ctx, "cdWorkflowService.SubmitWorkflow") + cdStageWorkflowRequest.Pipeline = pipeline + cdStageWorkflowRequest.Env = env + cdStageWorkflowRequest.Type = bean3.CD_WORKFLOW_PIPELINE_TYPE + _, err = impl.cdWorkflowService.SubmitWorkflow(cdStageWorkflowRequest) + span.End() + err = impl.sendPreStageNotification(ctx, cdWf, pipeline) + if err != nil { + return err + } + //creating cd config history entry + _, span = otel.Tracer("orchestrator").Start(ctx, "prePostCdScriptHistoryService.CreatePrePostCdScriptHistory") + err = impl.prePostCdScriptHistoryService.CreatePrePostCdScriptHistory(pipeline, nil, repository3.PRE_CD_TYPE, true, triggeredBy, triggeredAt) + span.End() + if err != nil { + impl.logger.Errorw("error in creating pre cd script entry", "err", err, "pipeline", pipeline) + return err + } + return nil +} + +func (impl *TriggerServiceImpl) SetCopyContainerImagePluginDataInWorkflowRequest(cdStageWorkflowRequest *types.WorkflowRequest, pipelineId int, pipelineStage string, artifact *repository.CiArtifact) ([]int, error) { + copyContainerImagePluginId, err := impl.globalPluginService.GetRefPluginIdByRefPluginName(pipeline.COPY_CONTAINER_IMAGE) + var imagePathReservationIds []int + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in getting copyContainerImage plugin id", "err", err) + return imagePathReservationIds, err + } + for _, step := range cdStageWorkflowRequest.PrePostDeploySteps { + if copyContainerImagePluginId != 0 && step.RefPluginId == copyContainerImagePluginId { + var pipelineStageEntityType int + if pipelineStage == types.PRE { + pipelineStageEntityType = bean3.EntityTypePreCD + } else { + pipelineStageEntityType = bean3.EntityTypePostCD + } + customTagId := -1 + var DockerImageTag string + + customTag, err := impl.customTagService.GetActiveCustomTagByEntityKeyAndValue(pipelineStageEntityType, strconv.Itoa(pipelineId)) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in fetching custom tag data", "err", err) + return imagePathReservationIds, err + } + + if !customTag.Enabled { + // case when custom tag is not configured - source image tag will be taken as docker image tag + pluginTriggerImageSplit := strings.Split(artifact.Image, ":") + DockerImageTag = pluginTriggerImageSplit[len(pluginTriggerImageSplit)-1] + } else { + // for copyContainerImage plugin parse destination images and save its data in image path reservation table + customTagDbObject, customDockerImageTag, err := impl.customTagService.GetCustomTag(pipelineStageEntityType, strconv.Itoa(pipelineId)) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in fetching custom tag by entity key and value for CD", "err", err) + return imagePathReservationIds, err + } + if customTagDbObject != nil && customTagDbObject.Id > 0 { + customTagId = customTagDbObject.Id + } + DockerImageTag = customDockerImageTag + } + + var sourceDockerRegistryId string + if artifact.DataSource == repository.PRE_CD || artifact.DataSource == repository.POST_CD || artifact.DataSource == repository.POST_CI { + if artifact.CredentialsSourceType == repository.GLOBAL_CONTAINER_REGISTRY { + sourceDockerRegistryId = artifact.CredentialSourceValue + } + } else { + sourceDockerRegistryId = cdStageWorkflowRequest.DockerRegistryId + } + registryDestinationImageMap, registryCredentialMap, err := impl.pluginInputVariableParser.HandleCopyContainerImagePluginInputVariables(step.InputVars, DockerImageTag, cdStageWorkflowRequest.CiArtifactDTO.Image, sourceDockerRegistryId) + if err != nil { + impl.logger.Errorw("error in parsing copyContainerImage input variable", "err", err) + return imagePathReservationIds, err + } + var destinationImages []string + for _, images := range registryDestinationImageMap { + for _, image := range images { + destinationImages = append(destinationImages, image) + } + } + // fetch already saved artifacts to check if they are already present + savedCIArtifacts, err := impl.ciArtifactRepository.FindCiArtifactByImagePaths(destinationImages) + if err != nil { + impl.logger.Errorw("error in fetching artifacts by image path", "err", err) + return imagePathReservationIds, err + } + if len(savedCIArtifacts) > 0 { + // if already present in ci artifact, return "image path already in use error" + return imagePathReservationIds, bean3.ErrImagePathInUse + } + imagePathReservationIds, err = impl.ReserveImagesGeneratedAtPlugin(customTagId, registryDestinationImageMap) + if err != nil { + impl.logger.Errorw("error in reserving image", "err", err) + return imagePathReservationIds, err + } + cdStageWorkflowRequest.RegistryDestinationImageMap = registryDestinationImageMap + cdStageWorkflowRequest.RegistryCredentialMap = registryCredentialMap + var pluginArtifactStage string + if pipelineStage == types.PRE { + pluginArtifactStage = repository.PRE_CD + } else { + pluginArtifactStage = repository.POST_CD + } + cdStageWorkflowRequest.PluginArtifactStage = pluginArtifactStage + } + } + return imagePathReservationIds, nil +} + +func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflowRunner, cdWf *pipelineConfig.CdWorkflow, cdPipeline *pipelineConfig.Pipeline, triggeredBy int32) (*types.WorkflowRequest, error) { + cdWorkflowConfig, err := impl.cdWorkflowRepository.FindConfigByPipelineId(cdPipeline.Id) + if err != nil && !util.IsErrNoRows(err) { + return nil, err + } + + workflowExecutor := runner.ExecutorType + + artifact, err := impl.ciArtifactRepository.Get(cdWf.CiArtifactId) + if err != nil { + return nil, err + } + // Migration of deprecated DataSource Type + if artifact.IsMigrationRequired() { + migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(artifact.Id) + if migrationErr != nil { + impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", artifact.Id) + } + } + ciMaterialInfo, err := repository.GetCiMaterialInfo(artifact.MaterialInfo, artifact.DataSource) + if err != nil { + impl.logger.Errorw("parsing error", "err", err) + return nil, err + } + + var ciProjectDetails []bean3.CiProjectDetails + var ciPipeline *pipelineConfig.CiPipeline + if cdPipeline.CiPipelineId > 0 { + ciPipeline, err = impl.ciPipelineRepository.FindById(cdPipeline.CiPipelineId) + if err != nil && !util.IsErrNoRows(err) { + impl.logger.Errorw("cannot find ciPipelineRequest", "err", err) + return nil, err + } + + for _, m := range ciPipeline.CiPipelineMaterials { + // git material should be active in this case + if m == nil || m.GitMaterial == nil || !m.GitMaterial.Active { + continue + } + var ciMaterialCurrent repository.CiMaterialInfo + for _, ciMaterial := range ciMaterialInfo { + if ciMaterial.Material.GitConfiguration.URL == m.GitMaterial.Url { + ciMaterialCurrent = ciMaterial + break + } + } + gitMaterial, err := impl.materialRepository.FindById(m.GitMaterialId) + if err != nil && !util.IsErrNoRows(err) { + impl.logger.Errorw("could not fetch git materials", "err", err) + return nil, err + } + + ciProjectDetail := bean3.CiProjectDetails{ + GitRepository: ciMaterialCurrent.Material.GitConfiguration.URL, + MaterialName: gitMaterial.Name, + CheckoutPath: gitMaterial.CheckoutPath, + FetchSubmodules: gitMaterial.FetchSubmodules, + SourceType: m.Type, + SourceValue: m.Value, + Type: string(m.Type), + GitOptions: bean3.GitOptions{ + UserName: gitMaterial.GitProvider.UserName, + Password: gitMaterial.GitProvider.Password, + SshPrivateKey: gitMaterial.GitProvider.SshPrivateKey, + AccessToken: gitMaterial.GitProvider.AccessToken, + AuthMode: gitMaterial.GitProvider.AuthMode, + }, + } + + if len(ciMaterialCurrent.Modifications) > 0 { + ciProjectDetail.CommitHash = ciMaterialCurrent.Modifications[0].Revision + ciProjectDetail.Author = ciMaterialCurrent.Modifications[0].Author + ciProjectDetail.GitTag = ciMaterialCurrent.Modifications[0].Tag + ciProjectDetail.Message = ciMaterialCurrent.Modifications[0].Message + commitTime, err := convert(ciMaterialCurrent.Modifications[0].ModifiedTime) + if err != nil { + return nil, err + } + ciProjectDetail.CommitTime = commitTime.Format(bean4.LayoutRFC3339) + } else if ciPipeline.PipelineType == bean3.CI_JOB { + // This has been done to resolve unmarshalling issue in ci-runner, in case of no commit time(eg- polling container images) + ciProjectDetail.CommitTime = time.Time{}.Format(bean4.LayoutRFC3339) + } else { + impl.logger.Debugw("devtronbug#1062", ciPipeline.Id, cdPipeline.Id) + return nil, fmt.Errorf("modifications not found for %d", ciPipeline.Id) + } + + // set webhook data + if m.Type == pipelineConfig.SOURCE_TYPE_WEBHOOK && len(ciMaterialCurrent.Modifications) > 0 { + webhookData := ciMaterialCurrent.Modifications[0].WebhookData + ciProjectDetail.WebhookData = pipelineConfig.WebhookData{ + Id: webhookData.Id, + EventActionType: webhookData.EventActionType, + Data: webhookData.Data, + } + } + + ciProjectDetails = append(ciProjectDetails, ciProjectDetail) + } + } + var stageYaml string + var deployStageWfr pipelineConfig.CdWorkflowRunner + var deployStageTriggeredByUserEmail string + var pipelineReleaseCounter int + var preDeploySteps []*bean3.StepObject + var postDeploySteps []*bean3.StepObject + var refPluginsData []*bean3.RefPluginObject + //if pipeline_stage_steps present for pre-CD or post-CD then no need to add stageYaml to cdWorkflowRequest in that + //case add PreDeploySteps and PostDeploySteps to cdWorkflowRequest, this is done for backward compatibility + pipelineStage, err := impl.pipelineStageService.GetCdStageByCdPipelineIdAndStageType(cdPipeline.Id, runner.WorkflowType.WorkflowTypeToStageType()) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in fetching CD pipeline stage", "cdPipelineId", cdPipeline.Id, "stage ", runner.WorkflowType.WorkflowTypeToStageType(), "err", err) + return nil, err + } + env, err := impl.envRepository.FindById(cdPipeline.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in getting environment by id", "err", err) + return nil, err + } + + //Scope will pick the environment of CD pipeline irrespective of in-cluster mode, + //since user sees the environment of the CD pipeline + scope := resourceQualifiers.Scope{ + AppId: cdPipeline.App.Id, + EnvId: env.Id, + ClusterId: env.ClusterId, + SystemMetadata: &resourceQualifiers.SystemMetadata{ + EnvironmentName: env.Name, + ClusterName: env.Cluster.ClusterName, + Namespace: env.Namespace, + Image: artifact.Image, + ImageTag: util3.GetImageTagFromImage(artifact.Image), + }, + } + if pipelineStage != nil { + var variableSnapshot map[string]string + if runner.WorkflowType == bean2.CD_WORKFLOW_TYPE_PRE { + //TODO: use const from pipeline.WorkflowService:95 + prePostAndRefPluginResponse, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(cdPipeline.Id, "preCD", scope) + if err != nil { + impl.logger.Errorw("error in getting pre, post & refPlugin steps data for wf request", "err", err, "cdPipelineId", cdPipeline.Id) + return nil, err + } + preDeploySteps = prePostAndRefPluginResponse.PreStageSteps + refPluginsData = prePostAndRefPluginResponse.RefPluginData + variableSnapshot = prePostAndRefPluginResponse.VariableSnapshot + } else if runner.WorkflowType == bean2.CD_WORKFLOW_TYPE_POST { + //TODO: use const from pipeline.WorkflowService:96 + prePostAndRefPluginResponse, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(cdPipeline.Id, "postCD", scope) + if err != nil { + impl.logger.Errorw("error in getting pre, post & refPlugin steps data for wf request", "err", err, "cdPipelineId", cdPipeline.Id) + return nil, err + } + postDeploySteps = prePostAndRefPluginResponse.PostStageSteps + refPluginsData = prePostAndRefPluginResponse.RefPluginData + variableSnapshot = prePostAndRefPluginResponse.VariableSnapshot + deployStageWfr, deployStageTriggeredByUserEmail, pipelineReleaseCounter, err = impl.getDeployStageDetails(cdPipeline.Id) + if err != nil { + impl.logger.Errorw("error in getting deployStageWfr, deployStageTriggeredByUser and pipelineReleaseCounter wf request", "err", err, "cdPipelineId", cdPipeline.Id) + return nil, err + } + } else { + return nil, fmt.Errorf("unsupported workflow triggerd") + } + + //Save Scoped VariableSnapshot + var variableSnapshotHistories = util4.GetBeansPtr( + repository5.GetSnapshotBean(runner.Id, repository5.HistoryReferenceTypeCDWORKFLOWRUNNER, variableSnapshot)) + if len(variableSnapshotHistories) > 0 { + err = impl.scopedVariableManager.SaveVariableHistoriesForTrigger(variableSnapshotHistories, runner.TriggeredBy) + if err != nil { + impl.logger.Errorf("Not able to save variable snapshot for CD trigger %s %d %s", err, runner.Id, variableSnapshot) + } + } + } else { + //in this case no plugin script is not present for this cdPipeline hence going with attaching preStage or postStage config + if runner.WorkflowType == bean2.CD_WORKFLOW_TYPE_PRE { + stageYaml = cdPipeline.PreStageConfig + } else if runner.WorkflowType == bean2.CD_WORKFLOW_TYPE_POST { + stageYaml = cdPipeline.PostStageConfig + deployStageWfr, deployStageTriggeredByUserEmail, pipelineReleaseCounter, err = impl.getDeployStageDetails(cdPipeline.Id) + if err != nil { + impl.logger.Errorw("error in getting deployStageWfr, deployStageTriggeredByUser and pipelineReleaseCounter wf request", "err", err, "cdPipelineId", cdPipeline.Id) + return nil, err + } + + } else { + return nil, fmt.Errorf("unsupported workflow triggerd") + } + } + + digestConfigurationRequest := imageDigestPolicy.DigestPolicyConfigurationRequest{PipelineId: cdPipeline.Id} + digestPolicyConfigurations, err := impl.imageDigestPolicyService.GetDigestPolicyConfigurations(digestConfigurationRequest) + if err != nil { + impl.logger.Errorw("error in checking if isImageDigestPolicyConfiguredForPipeline", "err", err, "pipelineId", cdPipeline.Id) + return nil, err + } + image := artifact.Image + if digestPolicyConfigurations.UseDigestForTrigger() { + image = ReplaceImageTagWithDigest(image, artifact.ImageDigest) + } + cdStageWorkflowRequest := &types.WorkflowRequest{ + EnvironmentId: cdPipeline.EnvironmentId, + AppId: cdPipeline.AppId, + WorkflowId: cdWf.Id, + WorkflowRunnerId: runner.Id, + WorkflowNamePrefix: strconv.Itoa(runner.Id) + "-" + runner.Name, + WorkflowPrefixForLog: strconv.Itoa(cdWf.Id) + string(runner.WorkflowType) + "-" + runner.Name, + CdImage: impl.config.GetDefaultImage(), + CdPipelineId: cdWf.PipelineId, + TriggeredBy: triggeredBy, + StageYaml: stageYaml, + CiProjectDetails: ciProjectDetails, + Namespace: runner.Namespace, + ActiveDeadlineSeconds: impl.config.GetDefaultTimeout(), + CiArtifactDTO: types.CiArtifactDTO{ + Id: artifact.Id, + PipelineId: artifact.PipelineId, + Image: artifact.Image, + ImageDigest: artifact.ImageDigest, + MaterialInfo: artifact.MaterialInfo, + DataSource: artifact.DataSource, + WorkflowId: artifact.WorkflowId, + }, + OrchestratorHost: impl.config.OrchestratorHost, + OrchestratorToken: impl.config.OrchestratorToken, + CloudProvider: impl.config.CloudProvider, + WorkflowExecutor: workflowExecutor, + RefPlugins: refPluginsData, + Scope: scope, + } + + extraEnvVariables := make(map[string]string) + if env != nil { + extraEnvVariables[plugin.CD_PIPELINE_ENV_NAME_KEY] = env.Name + if env.Cluster != nil { + extraEnvVariables[plugin.CD_PIPELINE_CLUSTER_NAME_KEY] = env.Cluster.ClusterName + } + } + ciWf, err := impl.ciWorkflowRepository.FindLastTriggeredWorkflowByArtifactId(artifact.Id) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in getting ciWf by artifactId", "err", err, "artifactId", artifact.Id) + return nil, err + } + var webhookAndCiData *gitSensorClient.WebhookAndCiData + if ciWf != nil && ciWf.GitTriggers != nil { + i := 1 + var gitCommitEnvVariables []types.GitMetadata + + for ciPipelineMaterialId, gitTrigger := range ciWf.GitTriggers { + extraEnvVariables[fmt.Sprintf("%s_%d", GIT_COMMIT_HASH_PREFIX, i)] = gitTrigger.Commit + extraEnvVariables[fmt.Sprintf("%s_%d", GIT_SOURCE_TYPE_PREFIX, i)] = string(gitTrigger.CiConfigureSourceType) + extraEnvVariables[fmt.Sprintf("%s_%d", GIT_SOURCE_VALUE_PREFIX, i)] = gitTrigger.CiConfigureSourceValue + + gitCommitEnvVariables = append(gitCommitEnvVariables, types.GitMetadata{ + GitCommitHash: gitTrigger.Commit, + GitSourceType: string(gitTrigger.CiConfigureSourceType), + GitSourceValue: gitTrigger.CiConfigureSourceValue, + }) + + // CODE-BLOCK starts - store extra environment variables if webhook + if gitTrigger.CiConfigureSourceType == pipelineConfig.SOURCE_TYPE_WEBHOOK { + webhookDataId := gitTrigger.WebhookData.Id + if webhookDataId > 0 { + webhookDataRequest := &gitSensorClient.WebhookDataRequest{ + Id: webhookDataId, + CiPipelineMaterialId: ciPipelineMaterialId, + } + webhookAndCiData, err = impl.gitSensorGrpcClient.GetWebhookData(context.Background(), webhookDataRequest) + if err != nil { + impl.logger.Errorw("err while getting webhook data from git-sensor", "err", err, "webhookDataRequest", webhookDataRequest) + return nil, err + } + if webhookAndCiData != nil { + for extEnvVariableKey, extEnvVariableVal := range webhookAndCiData.ExtraEnvironmentVariables { + extraEnvVariables[extEnvVariableKey] = extEnvVariableVal + } + } + } + } + // CODE_BLOCK ends + + i++ + } + gitMetadata, err := json.Marshal(&gitCommitEnvVariables) + if err != nil { + impl.logger.Errorw("err while marshaling git metdata", "err", err) + return nil, err + } + extraEnvVariables[plugin.GIT_METADATA] = string(gitMetadata) + + extraEnvVariables[GIT_SOURCE_COUNT] = strconv.Itoa(len(ciWf.GitTriggers)) + } + + childCdIds, err := impl.appWorkflowRepository.FindChildCDIdsByParentCDPipelineId(cdPipeline.Id) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in getting child cdPipelineIds by parent cdPipelineId", "err", err, "parent cdPipelineId", cdPipeline.Id) + return nil, err + } + if len(childCdIds) > 0 { + childPipelines, err := impl.pipelineRepository.FindByIdsIn(childCdIds) + if err != nil { + impl.logger.Errorw("error in getting pipelines by ids", "err", err, "ids", childCdIds) + return nil, err + } + var childCdEnvVariables []types.ChildCdMetadata + for i, childPipeline := range childPipelines { + extraEnvVariables[fmt.Sprintf("%s_%d", CHILD_CD_ENV_NAME_PREFIX, i+1)] = childPipeline.Environment.Name + extraEnvVariables[fmt.Sprintf("%s_%d", CHILD_CD_CLUSTER_NAME_PREFIX, i+1)] = childPipeline.Environment.Cluster.ClusterName + + childCdEnvVariables = append(childCdEnvVariables, types.ChildCdMetadata{ + ChildCdEnvName: childPipeline.Environment.Name, + ChildCdClusterName: childPipeline.Environment.Cluster.ClusterName, + }) + } + childCdEnvVariablesMetadata, err := json.Marshal(&childCdEnvVariables) + if err != nil { + impl.logger.Errorw("err while marshaling childCdEnvVariables", "err", err) + return nil, err + } + extraEnvVariables[plugin.CHILD_CD_METADATA] = string(childCdEnvVariablesMetadata) + + extraEnvVariables[CHILD_CD_COUNT] = strconv.Itoa(len(childPipelines)) + } + if ciPipeline != nil && ciPipeline.Id > 0 { + extraEnvVariables["APP_NAME"] = ciPipeline.App.AppName + cdStageWorkflowRequest.DockerUsername = ciPipeline.CiTemplate.DockerRegistry.Username + cdStageWorkflowRequest.DockerPassword = ciPipeline.CiTemplate.DockerRegistry.Password + cdStageWorkflowRequest.AwsRegion = ciPipeline.CiTemplate.DockerRegistry.AWSRegion + cdStageWorkflowRequest.DockerConnection = ciPipeline.CiTemplate.DockerRegistry.Connection + cdStageWorkflowRequest.DockerCert = ciPipeline.CiTemplate.DockerRegistry.Cert + cdStageWorkflowRequest.AccessKey = ciPipeline.CiTemplate.DockerRegistry.AWSAccessKeyId + cdStageWorkflowRequest.SecretKey = ciPipeline.CiTemplate.DockerRegistry.AWSSecretAccessKey + cdStageWorkflowRequest.DockerRegistryType = string(ciPipeline.CiTemplate.DockerRegistry.RegistryType) + cdStageWorkflowRequest.DockerRegistryURL = ciPipeline.CiTemplate.DockerRegistry.RegistryURL + cdStageWorkflowRequest.DockerRegistryId = ciPipeline.CiTemplate.DockerRegistry.Id + cdStageWorkflowRequest.CiPipelineType = ciPipeline.PipelineType + } else if cdPipeline.AppId > 0 { + ciTemplate, err := impl.ciTemplateRepository.FindByAppId(cdPipeline.AppId) + if err != nil { + return nil, err + } + extraEnvVariables["APP_NAME"] = ciTemplate.App.AppName + cdStageWorkflowRequest.DockerUsername = ciTemplate.DockerRegistry.Username + cdStageWorkflowRequest.DockerPassword = ciTemplate.DockerRegistry.Password + cdStageWorkflowRequest.AwsRegion = ciTemplate.DockerRegistry.AWSRegion + cdStageWorkflowRequest.DockerConnection = ciTemplate.DockerRegistry.Connection + cdStageWorkflowRequest.DockerCert = ciTemplate.DockerRegistry.Cert + cdStageWorkflowRequest.AccessKey = ciTemplate.DockerRegistry.AWSAccessKeyId + cdStageWorkflowRequest.SecretKey = ciTemplate.DockerRegistry.AWSSecretAccessKey + cdStageWorkflowRequest.DockerRegistryType = string(ciTemplate.DockerRegistry.RegistryType) + cdStageWorkflowRequest.DockerRegistryURL = ciTemplate.DockerRegistry.RegistryURL + appLabels, err := impl.appLabelRepository.FindAllByAppId(cdPipeline.AppId) + cdStageWorkflowRequest.DockerRegistryId = ciTemplate.DockerRegistry.Id + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in getting labels by appId", "err", err, "appId", cdPipeline.AppId) + return nil, err + } + var appLabelEnvVariables []types.AppLabelMetadata + for i, appLabel := range appLabels { + extraEnvVariables[fmt.Sprintf("%s_%d", APP_LABEL_KEY_PREFIX, i+1)] = appLabel.Key + extraEnvVariables[fmt.Sprintf("%s_%d", APP_LABEL_VALUE_PREFIX, i+1)] = appLabel.Value + appLabelEnvVariables = append(appLabelEnvVariables, types.AppLabelMetadata{ + AppLabelKey: appLabel.Key, + AppLabelValue: appLabel.Value, + }) + } + if len(appLabels) > 0 { + extraEnvVariables[APP_LABEL_COUNT] = strconv.Itoa(len(appLabels)) + appLabelEnvVariablesMetadata, err := json.Marshal(&appLabelEnvVariables) + if err != nil { + impl.logger.Errorw("err while marshaling appLabelEnvVariables", "err", err) + return nil, err + } + extraEnvVariables[plugin.APP_LABEL_METADATA] = string(appLabelEnvVariablesMetadata) + + } + } + cdStageWorkflowRequest.ExtraEnvironmentVariables = extraEnvVariables + cdStageWorkflowRequest.DeploymentTriggerTime = deployStageWfr.StartedOn + cdStageWorkflowRequest.DeploymentTriggeredBy = deployStageTriggeredByUserEmail + + if pipelineReleaseCounter > 0 { + cdStageWorkflowRequest.DeploymentReleaseCounter = pipelineReleaseCounter + } + if cdWorkflowConfig.CdCacheRegion == "" { + cdWorkflowConfig.CdCacheRegion = impl.config.GetDefaultCdLogsBucketRegion() + } + + if runner.WorkflowType == bean2.CD_WORKFLOW_TYPE_PRE { + //populate input variables of steps with extra env variables + setExtraEnvVariableInDeployStep(preDeploySteps, extraEnvVariables, webhookAndCiData) + cdStageWorkflowRequest.PrePostDeploySteps = preDeploySteps + } else if runner.WorkflowType == bean2.CD_WORKFLOW_TYPE_POST { + setExtraEnvVariableInDeployStep(postDeploySteps, extraEnvVariables, webhookAndCiData) + cdStageWorkflowRequest.PrePostDeploySteps = postDeploySteps + } + cdStageWorkflowRequest.BlobStorageConfigured = runner.BlobStorageEnabled + switch cdStageWorkflowRequest.CloudProvider { + case types.BLOB_STORAGE_S3: + //No AccessKey is used for uploading artifacts, instead IAM based auth is used + cdStageWorkflowRequest.CdCacheRegion = cdWorkflowConfig.CdCacheRegion + cdStageWorkflowRequest.CdCacheLocation = cdWorkflowConfig.CdCacheBucket + cdStageWorkflowRequest.ArtifactLocation, cdStageWorkflowRequest.CiArtifactBucket, cdStageWorkflowRequest.CiArtifactFileName = impl.buildArtifactLocationForS3(cdWorkflowConfig, cdWf, runner) + cdStageWorkflowRequest.BlobStorageS3Config = &blob_storage.BlobStorageS3Config{ + AccessKey: impl.config.BlobStorageS3AccessKey, + Passkey: impl.config.BlobStorageS3SecretKey, + EndpointUrl: impl.config.BlobStorageS3Endpoint, + IsInSecure: impl.config.BlobStorageS3EndpointInsecure, + CiCacheBucketName: cdWorkflowConfig.CdCacheBucket, + CiCacheRegion: cdWorkflowConfig.CdCacheRegion, + CiCacheBucketVersioning: impl.config.BlobStorageS3BucketVersioned, + CiArtifactBucketName: cdStageWorkflowRequest.CiArtifactBucket, + CiArtifactRegion: cdWorkflowConfig.CdCacheRegion, + CiArtifactBucketVersioning: impl.config.BlobStorageS3BucketVersioned, + CiLogBucketName: impl.config.GetDefaultBuildLogsBucket(), + CiLogRegion: impl.config.GetDefaultCdLogsBucketRegion(), + CiLogBucketVersioning: impl.config.BlobStorageS3BucketVersioned, + } + case types.BLOB_STORAGE_GCP: + cdStageWorkflowRequest.GcpBlobConfig = &blob_storage.GcpBlobConfig{ + CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, + ArtifactBucketName: impl.config.GetDefaultBuildLogsBucket(), + LogBucketName: impl.config.GetDefaultBuildLogsBucket(), + } + cdStageWorkflowRequest.ArtifactLocation = impl.buildDefaultArtifactLocation(cdWorkflowConfig, cdWf, runner) + cdStageWorkflowRequest.CiArtifactFileName = cdStageWorkflowRequest.ArtifactLocation + case types.BLOB_STORAGE_AZURE: + cdStageWorkflowRequest.AzureBlobConfig = &blob_storage.AzureBlobConfig{ + Enabled: true, + AccountName: impl.config.AzureAccountName, + BlobContainerCiCache: impl.config.AzureBlobContainerCiCache, + AccountKey: impl.config.AzureAccountKey, + BlobContainerCiLog: impl.config.AzureBlobContainerCiLog, + BlobContainerArtifact: impl.config.AzureBlobContainerCiLog, + } + cdStageWorkflowRequest.BlobStorageS3Config = &blob_storage.BlobStorageS3Config{ + EndpointUrl: impl.config.AzureGatewayUrl, + IsInSecure: impl.config.AzureGatewayConnectionInsecure, + CiLogBucketName: impl.config.AzureBlobContainerCiLog, + CiLogRegion: "", + AccessKey: impl.config.AzureAccountName, + } + cdStageWorkflowRequest.ArtifactLocation = impl.buildDefaultArtifactLocation(cdWorkflowConfig, cdWf, runner) + cdStageWorkflowRequest.CiArtifactFileName = cdStageWorkflowRequest.ArtifactLocation + default: + if impl.config.BlobStorageEnabled { + return nil, fmt.Errorf("blob storage %s not supported", cdStageWorkflowRequest.CloudProvider) + } + } + cdStageWorkflowRequest.DefaultAddressPoolBaseCidr = impl.config.GetDefaultAddressPoolBaseCidr() + cdStageWorkflowRequest.DefaultAddressPoolSize = impl.config.GetDefaultAddressPoolSize() + return cdStageWorkflowRequest, nil +} + +func (impl *TriggerServiceImpl) GetArtifactVulnerabilityStatus(artifact *repository.CiArtifact, cdPipeline *pipelineConfig.Pipeline, ctx context.Context) (bool, error) { + isVulnerable := false + if len(artifact.ImageDigest) > 0 { + var cveStores []*security.CveStore + _, span := otel.Tracer("orchestrator").Start(ctx, "scanResultRepository.FindByImageDigest") + imageScanResult, err := impl.scanResultRepository.FindByImageDigest(artifact.ImageDigest) + span.End() + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error fetching image digest", "digest", artifact.ImageDigest, "err", err) + return false, err + } + for _, item := range imageScanResult { + cveStores = append(cveStores, &item.CveStore) + } + _, span = otel.Tracer("orchestrator").Start(ctx, "cvePolicyRepository.GetBlockedCVEList") + if cdPipeline.Environment.ClusterId == 0 { + envDetails, err := impl.envRepository.FindById(cdPipeline.EnvironmentId) + if err != nil { + impl.logger.Errorw("error fetching cluster details by env, GetArtifactVulnerabilityStatus", "envId", cdPipeline.EnvironmentId, "err", err) + return false, err + } + cdPipeline.Environment = *envDetails + } + blockCveList, err := impl.cvePolicyRepository.GetBlockedCVEList(cveStores, cdPipeline.Environment.ClusterId, cdPipeline.EnvironmentId, cdPipeline.AppId, false) + span.End() + if err != nil { + impl.logger.Errorw("error while fetching env", "err", err) + return false, err + } + if len(blockCveList) > 0 { + isVulnerable = true + } + } + return isVulnerable, nil +} + +func (impl *TriggerServiceImpl) ReserveImagesGeneratedAtPlugin(customTagId int, registryImageMap map[string][]string) ([]int, error) { + var imagePathReservationIds []int + for _, images := range registryImageMap { + for _, image := range images { + imagePathReservationData, err := impl.customTagService.ReserveImagePath(image, customTagId) + if err != nil { + impl.logger.Errorw("Error in marking custom tag reserved", "err", err) + return imagePathReservationIds, err + } + if imagePathReservationData != nil { + imagePathReservationIds = append(imagePathReservationIds, imagePathReservationData.Id) + } + } + } + return imagePathReservationIds, nil +} + +func setExtraEnvVariableInDeployStep(deploySteps []*bean3.StepObject, extraEnvVariables map[string]string, webhookAndCiData *gitSensorClient.WebhookAndCiData) { + for _, deployStep := range deploySteps { + for variableKey, variableValue := range extraEnvVariables { + if isExtraVariableDynamic(variableKey, webhookAndCiData) && deployStep.StepType == "INLINE" { + extraInputVar := &bean3.VariableObject{ + Name: variableKey, + Format: "STRING", + Value: variableValue, + VariableType: bean3.VARIABLE_TYPE_REF_GLOBAL, + ReferenceVariableName: variableKey, + } + deployStep.InputVars = append(deployStep.InputVars, extraInputVar) + } + } + } +} + +func (impl *TriggerServiceImpl) getDeployStageDetails(pipelineId int) (pipelineConfig.CdWorkflowRunner, string, int, error) { + deployStageWfr := pipelineConfig.CdWorkflowRunner{} + //getting deployment pipeline latest wfr by pipelineId + deployStageWfr, err := impl.cdWorkflowRepository.FindLastStatusByPipelineIdAndRunnerType(pipelineId, bean2.CD_WORKFLOW_TYPE_DEPLOY) + if err != nil { + impl.logger.Errorw("error in getting latest status of deploy type wfr by pipelineId", "err", err, "pipelineId", pipelineId) + return deployStageWfr, "", 0, err + } + deployStageTriggeredByUserEmail, err := impl.userService.GetEmailById(deployStageWfr.TriggeredBy) + if err != nil { + impl.logger.Errorw("error in getting user email by id", "err", err, "userId", deployStageWfr.TriggeredBy) + return deployStageWfr, "", 0, err + } + pipelineReleaseCounter, err := impl.pipelineOverrideRepository.GetCurrentPipelineReleaseCounter(pipelineId) + if err != nil { + impl.logger.Errorw("error occurred while fetching latest release counter for pipeline", "pipelineId", pipelineId, "err", err) + return deployStageWfr, "", 0, err + } + return deployStageWfr, deployStageTriggeredByUserEmail, pipelineReleaseCounter, nil +} + +func (impl *TriggerServiceImpl) buildArtifactLocationForS3(cdWorkflowConfig *pipelineConfig.CdWorkflowConfig, cdWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner) (string, string, string) { + cdArtifactLocationFormat := cdWorkflowConfig.CdArtifactLocationFormat + if cdArtifactLocationFormat == "" { + cdArtifactLocationFormat = impl.config.GetArtifactLocationFormat() + } + if cdWorkflowConfig.LogsBucket == "" { + cdWorkflowConfig.LogsBucket = impl.config.GetDefaultBuildLogsBucket() + } + ArtifactLocation := fmt.Sprintf("s3://%s/"+impl.config.GetDefaultArtifactKeyPrefix()+"/"+cdArtifactLocationFormat, cdWorkflowConfig.LogsBucket, cdWf.Id, runner.Id) + artifactFileName := fmt.Sprintf(impl.config.GetDefaultArtifactKeyPrefix()+"/"+cdArtifactLocationFormat, cdWf.Id, runner.Id) + return ArtifactLocation, cdWorkflowConfig.LogsBucket, artifactFileName +} + +func (impl *TriggerServiceImpl) buildDefaultArtifactLocation(cdWorkflowConfig *pipelineConfig.CdWorkflowConfig, savedWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner) string { + cdArtifactLocationFormat := cdWorkflowConfig.CdArtifactLocationFormat + if cdArtifactLocationFormat == "" { + cdArtifactLocationFormat = impl.config.GetArtifactLocationFormat() + } + ArtifactLocation := fmt.Sprintf("%s/"+cdArtifactLocationFormat, impl.config.GetDefaultArtifactKeyPrefix(), savedWf.Id, runner.Id) + return ArtifactLocation +} + +func ReplaceImageTagWithDigest(image, digest string) string { + imageWithoutTag := strings.Split(image, ":")[0] + imageWithDigest := fmt.Sprintf("%s@%s", imageWithoutTag, digest) + return imageWithDigest +} + +func (impl *TriggerServiceImpl) sendPreStageNotification(ctx context.Context, cdWf *pipelineConfig.CdWorkflow, pipeline *pipelineConfig.Pipeline) error { + wfr, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(ctx, cdWf.Id, bean2.CD_WORKFLOW_TYPE_PRE) + if err != nil { + return err + } + + event := impl.eventFactory.Build(util2.Trigger, &pipeline.Id, pipeline.AppId, &pipeline.EnvironmentId, util2.CD) + impl.logger.Debugw("event PreStageTrigger", "event", event) + event = impl.eventFactory.BuildExtraCDData(event, &wfr, 0, bean2.CD_WORKFLOW_TYPE_PRE) + _, span := otel.Tracer("orchestrator").Start(ctx, "eventClient.WriteNotificationEvent") + _, evtErr := impl.eventClient.WriteNotificationEvent(event) + span.End() + if evtErr != nil { + impl.logger.Errorw("CD trigger event not sent", "error", evtErr) + } + return nil +} + +func isExtraVariableDynamic(variableName string, webhookAndCiData *gitSensorClient.WebhookAndCiData) bool { + if strings.Contains(variableName, GIT_COMMIT_HASH_PREFIX) || strings.Contains(variableName, GIT_SOURCE_TYPE_PREFIX) || strings.Contains(variableName, GIT_SOURCE_VALUE_PREFIX) || + strings.Contains(variableName, APP_LABEL_VALUE_PREFIX) || strings.Contains(variableName, APP_LABEL_KEY_PREFIX) || + strings.Contains(variableName, CHILD_CD_ENV_NAME_PREFIX) || strings.Contains(variableName, CHILD_CD_CLUSTER_NAME_PREFIX) || + strings.Contains(variableName, CHILD_CD_COUNT) || strings.Contains(variableName, APP_LABEL_COUNT) || strings.Contains(variableName, GIT_SOURCE_COUNT) || + webhookAndCiData != nil { + + return true + } + return false +} +func convert(ts string) (*time.Time, error) { + t, err := time.Parse(bean4.LayoutRFC3339, ts) + if err != nil { + return nil, err + } + return &t, nil +} diff --git a/pkg/deployment/trigger/devtronApps/TriggerService.go b/pkg/deployment/trigger/devtronApps/TriggerService.go index 02f1aff570..bf507b7dc0 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService.go +++ b/pkg/deployment/trigger/devtronApps/TriggerService.go @@ -13,8 +13,11 @@ import ( "github.com/devtron-labs/devtron/client/argocdServer" bean7 "github.com/devtron-labs/devtron/client/argocdServer/bean" client "github.com/devtron-labs/devtron/client/events" + gitSensorClient "github.com/devtron-labs/devtron/client/gitSensor" "github.com/devtron-labs/devtron/internal/middleware" + "github.com/devtron-labs/devtron/internal/sql/models" repository3 "github.com/devtron-labs/devtron/internal/sql/repository" + "github.com/devtron-labs/devtron/internal/sql/repository/appWorkflow" "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/sql/repository/security" @@ -22,17 +25,29 @@ import ( "github.com/devtron-labs/devtron/pkg/app" bean4 "github.com/devtron-labs/devtron/pkg/app/bean" "github.com/devtron-labs/devtron/pkg/app/status" + "github.com/devtron-labs/devtron/pkg/auth/user" bean2 "github.com/devtron-labs/devtron/pkg/bean" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/deployment/manifest" bean5 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/bean" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/helper" + "github.com/devtron-labs/devtron/pkg/eventProcessor/out" + "github.com/devtron-labs/devtron/pkg/imageDigestPolicy" + "github.com/devtron-labs/devtron/pkg/pipeline" + bean8 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/history" "github.com/devtron-labs/devtron/pkg/pipeline/repository" + "github.com/devtron-labs/devtron/pkg/pipeline/types" + "github.com/devtron-labs/devtron/pkg/plugin" "github.com/devtron-labs/devtron/pkg/sql" + "github.com/devtron-labs/devtron/pkg/variables" "github.com/devtron-labs/devtron/pkg/workflow/cd" util3 "github.com/devtron-labs/devtron/util" + "github.com/devtron-labs/devtron/util/argo" util2 "github.com/devtron-labs/devtron/util/event" + "github.com/devtron-labs/devtron/util/rbac" "github.com/go-pg/pg" "go.opentelemetry.io/otel" "go.uber.org/zap" @@ -40,11 +55,23 @@ import ( status2 "google.golang.org/grpc/status" "k8s.io/helm/pkg/proto/hapi/chart" "path" + "strconv" "strings" "time" ) type TriggerService interface { + TriggerPostStage(request bean.TriggerRequest) error + TriggerPreStage(request bean.TriggerRequest) error + + TriggerStageForBulk(triggerRequest bean.TriggerRequest) error + + ManualCdTrigger(triggerContext bean.TriggerContext, overrideRequest *bean3.ValuesOverrideRequest) (int, error) + TriggerAutomaticDeployment(request bean.TriggerRequest) error + + HandleCDTriggerRelease(overrideRequest *bean3.ValuesOverrideRequest, ctx context.Context, + triggeredAt time.Time, deployedBy int32) (releaseNo int, manifest []byte, err error) + TriggerRelease(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, ctx context.Context, triggeredAt time.Time, triggeredBy int32) (releaseNo int, manifest []byte, err error) @@ -53,22 +80,40 @@ type TriggerService interface { } type TriggerServiceImpl struct { - logger *zap.SugaredLogger - cdWorkflowCommonService cd.CdWorkflowCommonService - gitOpsManifestPushService app.GitOpsPushService - argoK8sClient argocdServer.ArgoK8sClient - ACDConfig *argocdServer.ACDConfig - argoClientWrapperService argocdServer.ArgoClientWrapperService - pipelineStatusTimelineService status.PipelineStatusTimelineService - chartTemplateService util.ChartTemplateService - eventFactory client.EventFactory - eventClient client.EventClient - globalEnvVariables *util3.GlobalEnvVariables + logger *zap.SugaredLogger + cdWorkflowCommonService cd.CdWorkflowCommonService + gitOpsManifestPushService app.GitOpsPushService + argoK8sClient argocdServer.ArgoK8sClient + ACDConfig *argocdServer.ACDConfig + argoClientWrapperService argocdServer.ArgoClientWrapperService + pipelineStatusTimelineService status.PipelineStatusTimelineService + chartTemplateService util.ChartTemplateService + eventFactory client.EventFactory + eventClient client.EventClient + globalEnvVariables *util3.GlobalEnvVariables + workflowEventPublishService out.WorkflowEventPublishService + manifestCreationService manifest.ManifestCreationService + deployedConfigurationHistoryService history.DeployedConfigurationHistoryService + argoUserService argo.ArgoUserService + pipelineStageService pipeline.PipelineStageService + globalPluginService plugin.GlobalPluginService + customTagService pipeline.CustomTagService + pluginInputVariableParser pipeline.PluginInputVariableParser + prePostCdScriptHistoryService history.PrePostCdScriptHistoryService + scopedVariableManager variables.ScopedVariableCMCSManager + cdWorkflowService pipeline.WorkflowService + imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService + userService user.UserService + gitSensorGrpcClient gitSensorClient.Client + config *types.CdConfig helmAppService client2.HelmAppService + enforcerUtil rbac.EnforcerUtil helmAppClient gRPC.HelmAppClient //TODO refactoring: use helm app service instead + scanResultRepository security.ImageScanResultRepository + cvePolicyRepository security.CvePolicyRepository ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository imageScanHistoryRepository security.ImageScanHistoryRepository imageScanDeployInfoRepository security.ImageScanDeployInfoRepository @@ -77,6 +122,14 @@ type TriggerServiceImpl struct { manifestPushConfigRepository repository.ManifestPushConfigRepository chartRepository chartRepoRepository.ChartRepository envRepository repository2.EnvironmentRepository + cdWorkflowRepository pipelineConfig.CdWorkflowRepository + ciWorkflowRepository pipelineConfig.CiWorkflowRepository + ciArtifactRepository repository3.CiArtifactRepository + ciTemplateRepository pipelineConfig.CiTemplateRepository + materialRepository pipelineConfig.MaterialRepository + appLabelRepository pipelineConfig.AppLabelRepository + ciPipelineRepository pipelineConfig.CiPipelineRepository + appWorkflowRepository appWorkflow.AppWorkflowRepository } func NewTriggerServiceImpl(logger *zap.SugaredLogger, cdWorkflowCommonService cd.CdWorkflowCommonService, @@ -86,11 +139,28 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, cdWorkflowCommonService cd argoClientWrapperService argocdServer.ArgoClientWrapperService, pipelineStatusTimelineService status.PipelineStatusTimelineService, chartTemplateService util.ChartTemplateService, + workflowEventPublishService out.WorkflowEventPublishService, + manifestCreationService manifest.ManifestCreationService, + deployedConfigurationHistoryService history.DeployedConfigurationHistoryService, + argoUserService argo.ArgoUserService, + pipelineStageService pipeline.PipelineStageService, + globalPluginService plugin.GlobalPluginService, + customTagService pipeline.CustomTagService, + pluginInputVariableParser pipeline.PluginInputVariableParser, + prePostCdScriptHistoryService history.PrePostCdScriptHistoryService, + scopedVariableManager variables.ScopedVariableCMCSManager, + cdWorkflowService pipeline.WorkflowService, + imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService, + userService user.UserService, + gitSensorGrpcClient gitSensorClient.Client, helmAppService client2.HelmAppService, + enforcerUtil rbac.EnforcerUtil, helmAppClient gRPC.HelmAppClient, eventFactory client.EventFactory, eventClient client.EventClient, globalEnvVariables *util3.GlobalEnvVariables, + scanResultRepository security.ImageScanResultRepository, + cvePolicyRepository security.CvePolicyRepository, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, imageScanHistoryRepository security.ImageScanHistoryRepository, imageScanDeployInfoRepository security.ImageScanDeployInfoRepository, @@ -98,32 +168,542 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, cdWorkflowCommonService cd pipelineOverrideRepository chartConfig.PipelineOverrideRepository, manifestPushConfigRepository repository.ManifestPushConfigRepository, chartRepository chartRepoRepository.ChartRepository, - envRepository repository2.EnvironmentRepository) *TriggerServiceImpl { - return &TriggerServiceImpl{ - logger: logger, - cdWorkflowCommonService: cdWorkflowCommonService, - gitOpsManifestPushService: gitOpsManifestPushService, - argoK8sClient: argoK8sClient, - ACDConfig: ACDConfig, - argoClientWrapperService: argoClientWrapperService, - pipelineStatusTimelineService: pipelineStatusTimelineService, - chartTemplateService: chartTemplateService, - helmAppService: helmAppService, - eventFactory: eventFactory, - eventClient: eventClient, - globalEnvVariables: globalEnvVariables, - helmAppClient: helmAppClient, - ciPipelineMaterialRepository: ciPipelineMaterialRepository, - imageScanHistoryRepository: imageScanHistoryRepository, - imageScanDeployInfoRepository: imageScanDeployInfoRepository, - pipelineRepository: pipelineRepository, - pipelineOverrideRepository: pipelineOverrideRepository, - manifestPushConfigRepository: manifestPushConfigRepository, - chartRepository: chartRepository, - envRepository: envRepository, + envRepository repository2.EnvironmentRepository, + cdWorkflowRepository pipelineConfig.CdWorkflowRepository, + ciWorkflowRepository pipelineConfig.CiWorkflowRepository, + ciArtifactRepository repository3.CiArtifactRepository, + ciTemplateRepository pipelineConfig.CiTemplateRepository, + materialRepository pipelineConfig.MaterialRepository, + appLabelRepository pipelineConfig.AppLabelRepository, + ciPipelineRepository pipelineConfig.CiPipelineRepository, + appWorkflowRepository appWorkflow.AppWorkflowRepository) (*TriggerServiceImpl, error) { + impl := &TriggerServiceImpl{ + logger: logger, + cdWorkflowCommonService: cdWorkflowCommonService, + gitOpsManifestPushService: gitOpsManifestPushService, + argoK8sClient: argoK8sClient, + ACDConfig: ACDConfig, + argoClientWrapperService: argoClientWrapperService, + pipelineStatusTimelineService: pipelineStatusTimelineService, + chartTemplateService: chartTemplateService, + workflowEventPublishService: workflowEventPublishService, + manifestCreationService: manifestCreationService, + deployedConfigurationHistoryService: deployedConfigurationHistoryService, + argoUserService: argoUserService, + pipelineStageService: pipelineStageService, + globalPluginService: globalPluginService, + customTagService: customTagService, + pluginInputVariableParser: pluginInputVariableParser, + prePostCdScriptHistoryService: prePostCdScriptHistoryService, + scopedVariableManager: scopedVariableManager, + cdWorkflowService: cdWorkflowService, + imageDigestPolicyService: imageDigestPolicyService, + userService: userService, + gitSensorGrpcClient: gitSensorGrpcClient, + helmAppService: helmAppService, + enforcerUtil: enforcerUtil, + eventFactory: eventFactory, + eventClient: eventClient, + globalEnvVariables: globalEnvVariables, + helmAppClient: helmAppClient, + scanResultRepository: scanResultRepository, + cvePolicyRepository: cvePolicyRepository, + ciPipelineMaterialRepository: ciPipelineMaterialRepository, + imageScanHistoryRepository: imageScanHistoryRepository, + imageScanDeployInfoRepository: imageScanDeployInfoRepository, + pipelineRepository: pipelineRepository, + pipelineOverrideRepository: pipelineOverrideRepository, + manifestPushConfigRepository: manifestPushConfigRepository, + chartRepository: chartRepository, + envRepository: envRepository, + cdWorkflowRepository: cdWorkflowRepository, + ciWorkflowRepository: ciWorkflowRepository, + ciArtifactRepository: ciArtifactRepository, + ciTemplateRepository: ciTemplateRepository, + materialRepository: materialRepository, + appLabelRepository: appLabelRepository, + ciPipelineRepository: ciPipelineRepository, + appWorkflowRepository: appWorkflowRepository, + } + config, err := types.GetCdConfig() + if err != nil { + return nil, err + } + impl.config = config + return impl, nil +} + +func (impl *TriggerServiceImpl) TriggerStageForBulk(triggerRequest bean.TriggerRequest) error { + + preStage, err := impl.pipelineStageService.GetCdStageByCdPipelineIdAndStageType(triggerRequest.Pipeline.Id, repository.PIPELINE_STAGE_TYPE_PRE_CD) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in fetching CD pipeline stage", "cdPipelineId", triggerRequest.Pipeline.Id, "stage ", repository.PIPELINE_STAGE_TYPE_PRE_CD, "err", err) + return err + } + + //handle corrupt data (https://github.com/devtron-labs/devtron/issues/3826) + err, deleted := impl.deleteCorruptedPipelineStage(preStage, triggerRequest.TriggeredBy) + if err != nil { + impl.logger.Errorw("error in deleteCorruptedPipelineStage ", "cdPipelineId", triggerRequest.Pipeline.Id, "err", err, "preStage", preStage, "triggeredBy", triggerRequest.TriggeredBy) + return err + } + + triggerRequest.TriggerContext.Context = context.Background() + if len(triggerRequest.Pipeline.PreStageConfig) > 0 || (preStage != nil && !deleted) { + //pre stage exists + impl.logger.Debugw("trigger pre stage for pipeline", "artifactId", triggerRequest.Artifact.Id, "pipelineId", triggerRequest.Pipeline.Id) + triggerRequest.RefCdWorkflowRunnerId = 0 + err = impl.TriggerPreStage(triggerRequest) // TODO handle error here + return err + } else { + // trigger deployment + impl.logger.Debugw("trigger cd for pipeline", "artifactId", triggerRequest.Artifact.Id, "pipelineId", triggerRequest.Pipeline.Id) + err = impl.TriggerAutomaticDeployment(triggerRequest) + return err } } +// TODO: write a wrapper to handle auto and manual trigger +func (impl *TriggerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerContext, overrideRequest *bean3.ValuesOverrideRequest) (int, error) { + //setting triggeredAt variable to have consistent data for various audit log places in db for deployment time + triggeredAt := time.Now() + releaseId := 0 + ctx := triggerContext.Context + var err error + _, span := otel.Tracer("orchestrator").Start(ctx, "pipelineRepository.FindById") + cdPipeline, err := impl.pipelineRepository.FindById(overrideRequest.PipelineId) + span.End() + if err != nil { + impl.logger.Errorw("manual trigger request with invalid pipelineId, ManualCdTrigger", "pipelineId", overrideRequest.PipelineId, "err", err) + return 0, err + } + SetPipelineFieldsInOverrideRequest(overrideRequest, cdPipeline) + + switch overrideRequest.CdWorkflowType { + case bean3.CD_WORKFLOW_TYPE_PRE: + _, span = otel.Tracer("orchestrator").Start(ctx, "ciArtifactRepository.Get") + artifact, err := impl.ciArtifactRepository.Get(overrideRequest.CiArtifactId) + span.End() + if err != nil { + impl.logger.Errorw("error in getting CiArtifact", "CiArtifactId", overrideRequest.CiArtifactId, "err", err) + return 0, err + } + // Migration of deprecated DataSource Type + if artifact.IsMigrationRequired() { + migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(artifact.Id) + if migrationErr != nil { + impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", artifact.Id) + } + } + _, span = otel.Tracer("orchestrator").Start(ctx, "TriggerPreStage") + triggerRequest := bean.TriggerRequest{ + CdWf: nil, + Artifact: artifact, + Pipeline: cdPipeline, + TriggeredBy: overrideRequest.UserId, + ApplyAuth: false, + TriggerContext: triggerContext, + RefCdWorkflowRunnerId: 0, + } + err = impl.TriggerPreStage(triggerRequest) + span.End() + if err != nil { + impl.logger.Errorw("error in TriggerPreStage, ManualCdTrigger", "err", err) + return 0, err + } + case bean3.CD_WORKFLOW_TYPE_DEPLOY: + if overrideRequest.DeploymentType == models.DEPLOYMENTTYPE_UNKNOWN { + overrideRequest.DeploymentType = models.DEPLOYMENTTYPE_DEPLOY + } + + cdWf, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(ctx, overrideRequest.CdWorkflowId, bean3.CD_WORKFLOW_TYPE_PRE) + if err != nil && !util.IsErrNoRows(err) { + impl.logger.Errorw("error in getting cdWorkflow, ManualCdTrigger", "CdWorkflowId", overrideRequest.CdWorkflowId, "err", err) + return 0, err + } + + cdWorkflowId := cdWf.CdWorkflowId + if cdWf.CdWorkflowId == 0 { + cdWf := &pipelineConfig.CdWorkflow{ + CiArtifactId: overrideRequest.CiArtifactId, + PipelineId: overrideRequest.PipelineId, + AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: overrideRequest.UserId, UpdatedOn: triggeredAt, UpdatedBy: overrideRequest.UserId}, + } + err := impl.cdWorkflowRepository.SaveWorkFlow(ctx, cdWf) + if err != nil { + impl.logger.Errorw("error in creating cdWorkflow, ManualCdTrigger", "PipelineId", overrideRequest.PipelineId, "err", err) + return 0, err + } + cdWorkflowId = cdWf.Id + } + + runner := &pipelineConfig.CdWorkflowRunner{ + Name: cdPipeline.Name, + WorkflowType: bean3.CD_WORKFLOW_TYPE_DEPLOY, + ExecutorType: pipelineConfig.WORKFLOW_EXECUTOR_TYPE_AWF, + Status: pipelineConfig.WorkflowInitiated, //deployment Initiated for manual trigger + TriggeredBy: overrideRequest.UserId, + StartedOn: triggeredAt, + Namespace: impl.config.GetDefaultNamespace(), + CdWorkflowId: cdWorkflowId, + AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: overrideRequest.UserId, UpdatedOn: triggeredAt, UpdatedBy: overrideRequest.UserId}, + ReferenceId: triggerContext.ReferenceId, + } + savedWfr, err := impl.cdWorkflowRepository.SaveWorkFlowRunner(runner) + overrideRequest.WfrId = savedWfr.Id + if err != nil { + impl.logger.Errorw("err in creating cdWorkflowRunner, ManualCdTrigger", "cdWorkflowId", cdWorkflowId, "err", err) + return 0, err + } + runner.CdWorkflow = &pipelineConfig.CdWorkflow{ + Pipeline: cdPipeline, + } + overrideRequest.CdWorkflowId = cdWorkflowId + // creating cd pipeline status timeline for deployment initialisation + timeline := impl.pipelineStatusTimelineService.GetTimelineDbObjectByTimelineStatusAndTimelineDescription(savedWfr.Id, 0, pipelineConfig.TIMELINE_STATUS_DEPLOYMENT_INITIATED, pipelineConfig.TIMELINE_DESCRIPTION_DEPLOYMENT_INITIATED, overrideRequest.UserId, time.Now()) + _, span = otel.Tracer("orchestrator").Start(ctx, "cdPipelineStatusTimelineRepo.SaveTimelineForACDHelmApps") + err = impl.pipelineStatusTimelineService.SaveTimeline(timeline, nil, false) + + span.End() + if err != nil { + impl.logger.Errorw("error in creating timeline status for deployment initiation, ManualCdTrigger", "err", err, "timeline", timeline) + } + + //checking vulnerability for deploying image + _, span = otel.Tracer("orchestrator").Start(ctx, "ciArtifactRepository.Get") + artifact, err := impl.ciArtifactRepository.Get(overrideRequest.CiArtifactId) + span.End() + if err != nil { + impl.logger.Errorw("error in getting ciArtifact, ManualCdTrigger", "CiArtifactId", overrideRequest.CiArtifactId, "err", err) + return 0, err + } + // Migration of deprecated DataSource Type + if artifact.IsMigrationRequired() { + migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(artifact.Id) + if migrationErr != nil { + impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", artifact.Id) + } + } + isVulnerable, err := impl.GetArtifactVulnerabilityStatus(artifact, cdPipeline, ctx) + if err != nil { + impl.logger.Errorw("error in getting Artifact vulnerability status, ManualCdTrigger", "err", err) + return 0, err + } + + if isVulnerable == true { + // if image vulnerable, update timeline status and return + if err = impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(runner, errors.New(pipelineConfig.FOUND_VULNERABILITY), overrideRequest.UserId); err != nil { + impl.logger.Errorw("error while updating current runner status to failed, TriggerDeployment", "wfrId", runner.Id, "err", err) + } + return 0, fmt.Errorf("found vulnerability for image digest %s", artifact.ImageDigest) + } + + // Deploy the release + _, span = otel.Tracer("orchestrator").Start(ctx, "appService.TriggerRelease") + var releaseErr error + releaseId, _, releaseErr = impl.HandleCDTriggerRelease(overrideRequest, ctx, triggeredAt, overrideRequest.UserId) + span.End() + // if releaseErr found, then the mark current deployment Failed and return + if releaseErr != nil { + err := impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(runner, releaseErr, overrideRequest.UserId) + if err != nil { + impl.logger.Errorw("error while updating current runner status to failed, updatePreviousDeploymentStatus", "cdWfr", runner.Id, "err", err) + } + return 0, releaseErr + } + + // skip updatePreviousDeploymentStatus if Async Install is enabled; handled inside SubscribeDevtronAsyncHelmInstallRequest + if !impl.IsDevtronAsyncInstallModeEnabled(cdPipeline.DeploymentAppType) { + // Update previous deployment runner status (in transaction): Failed + _, span = otel.Tracer("orchestrator").Start(ctx, "updatePreviousDeploymentStatus") + err1 := impl.cdWorkflowCommonService.UpdatePreviousDeploymentStatus(runner, cdPipeline.Id, triggeredAt, overrideRequest.UserId) + span.End() + if err1 != nil { + impl.logger.Errorw("error while update previous cd workflow runners, ManualCdTrigger", "err", err, "runner", runner, "pipelineId", cdPipeline.Id) + return 0, err1 + } + } + + if overrideRequest.DeploymentAppType == util.PIPELINE_DEPLOYMENT_TYPE_MANIFEST_DOWNLOAD { + runner := &pipelineConfig.CdWorkflowRunner{ + Id: runner.Id, + Name: cdPipeline.Name, + WorkflowType: bean3.CD_WORKFLOW_TYPE_DEPLOY, + ExecutorType: pipelineConfig.WORKFLOW_EXECUTOR_TYPE_AWF, + TriggeredBy: overrideRequest.UserId, + StartedOn: triggeredAt, + Status: pipelineConfig.WorkflowSucceeded, + Namespace: impl.config.GetDefaultNamespace(), + CdWorkflowId: overrideRequest.CdWorkflowId, + AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: overrideRequest.UserId, UpdatedOn: triggeredAt, UpdatedBy: overrideRequest.UserId}, + } + updateErr := impl.cdWorkflowRepository.UpdateWorkFlowRunner(runner) + if updateErr != nil { + impl.logger.Errorw("error in updating runner for manifest_download type, ManualCdTrigger", "CdWorkflowId", overrideRequest.CdWorkflowId, "err", err) + return 0, updateErr + } + } + + case bean3.CD_WORKFLOW_TYPE_POST: + cdWfRunner, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(ctx, overrideRequest.CdWorkflowId, bean3.CD_WORKFLOW_TYPE_DEPLOY) + if err != nil && !util.IsErrNoRows(err) { + impl.logger.Errorw("err in getting cdWorkflowRunner, ManualCdTrigger", "cdWorkflowId", overrideRequest.CdWorkflowId, "err", err) + return 0, err + } + + var cdWf *pipelineConfig.CdWorkflow + if cdWfRunner.CdWorkflowId == 0 { + cdWf = &pipelineConfig.CdWorkflow{ + CiArtifactId: overrideRequest.CiArtifactId, + PipelineId: overrideRequest.PipelineId, + AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: overrideRequest.UserId, UpdatedOn: triggeredAt, UpdatedBy: overrideRequest.UserId}, + } + err := impl.cdWorkflowRepository.SaveWorkFlow(ctx, cdWf) + if err != nil { + impl.logger.Errorw("error in creating cdWorkflow, ManualCdTrigger", "CdWorkflowId", overrideRequest.CdWorkflowId, "err", err) + return 0, err + } + } else { + _, span = otel.Tracer("orchestrator").Start(ctx, "cdWorkflowRepository.FindById") + cdWf, err = impl.cdWorkflowRepository.FindById(overrideRequest.CdWorkflowId) + span.End() + if err != nil && !util.IsErrNoRows(err) { + impl.logger.Errorw("error in getting cdWorkflow, ManualCdTrigger", "CdWorkflowId", overrideRequest.CdWorkflowId, "err", err) + return 0, err + } + } + _, span = otel.Tracer("orchestrator").Start(ctx, "TriggerPostStage") + triggerRequest := bean.TriggerRequest{ + CdWf: cdWf, + Pipeline: cdPipeline, + TriggeredBy: overrideRequest.UserId, + RefCdWorkflowRunnerId: 0, + TriggerContext: triggerContext, + } + err = impl.TriggerPostStage(triggerRequest) + span.End() + if err != nil { + impl.logger.Errorw("error in TriggerPostStage, ManualCdTrigger", "CdWorkflowId", cdWf.Id, "err", err) + return 0, err + } + default: + impl.logger.Errorw("invalid CdWorkflowType, ManualCdTrigger", "CdWorkflowType", overrideRequest.CdWorkflowType, "err", err) + return 0, fmt.Errorf("invalid CdWorkflowType %s for the trigger request", string(overrideRequest.CdWorkflowType)) + } + + return releaseId, err +} + +// TODO: write a wrapper to handle auto and manual trigger +func (impl *TriggerServiceImpl) TriggerAutomaticDeployment(request bean.TriggerRequest) error { + //in case of manual trigger auth is already applied and for auto triggers there is no need for auth check here + triggeredBy := request.TriggeredBy + pipeline := request.Pipeline + artifact := request.Artifact + + //setting triggeredAt variable to have consistent data for various audit log places in db for deployment time + triggeredAt := time.Now() + cdWf := request.CdWf + + if cdWf == nil || (cdWf != nil && cdWf.CiArtifactId != artifact.Id) { + // cdWf != nil && cdWf.CiArtifactId != artifact.Id for auto trigger case when deployment is triggered with image generated by plugin + cdWf = &pipelineConfig.CdWorkflow{ + CiArtifactId: artifact.Id, + PipelineId: pipeline.Id, + AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: 1, UpdatedOn: triggeredAt, UpdatedBy: 1}, + } + err := impl.cdWorkflowRepository.SaveWorkFlow(context.Background(), cdWf) + if err != nil { + return err + } + } + + runner := &pipelineConfig.CdWorkflowRunner{ + Name: pipeline.Name, + WorkflowType: bean3.CD_WORKFLOW_TYPE_DEPLOY, + ExecutorType: pipelineConfig.WORKFLOW_EXECUTOR_TYPE_SYSTEM, + Status: pipelineConfig.WorkflowInitiated, // deployment Initiated for auto trigger + TriggeredBy: 1, + StartedOn: triggeredAt, + Namespace: impl.config.GetDefaultNamespace(), + CdWorkflowId: cdWf.Id, + AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: triggeredBy, UpdatedOn: triggeredAt, UpdatedBy: triggeredBy}, + ReferenceId: request.TriggerContext.ReferenceId, + } + savedWfr, err := impl.cdWorkflowRepository.SaveWorkFlowRunner(runner) + if err != nil { + return err + } + runner.CdWorkflow = &pipelineConfig.CdWorkflow{ + Pipeline: pipeline, + } + // creating cd pipeline status timeline for deployment initialisation + timeline := &pipelineConfig.PipelineStatusTimeline{ + CdWorkflowRunnerId: runner.Id, + Status: pipelineConfig.TIMELINE_STATUS_DEPLOYMENT_INITIATED, + StatusDetail: "Deployment initiated successfully.", + StatusTime: time.Now(), + AuditLog: sql.AuditLog{ + CreatedBy: 1, + CreatedOn: time.Now(), + UpdatedBy: 1, + UpdatedOn: time.Now(), + }, + } + isAppStore := false + err = impl.pipelineStatusTimelineService.SaveTimeline(timeline, nil, isAppStore) + if err != nil { + impl.logger.Errorw("error in creating timeline status for deployment initiation", "err", err, "timeline", timeline) + } + //checking vulnerability for deploying image + isVulnerable := false + if len(artifact.ImageDigest) > 0 { + var cveStores []*security.CveStore + imageScanResult, err := impl.scanResultRepository.FindByImageDigest(artifact.ImageDigest) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error fetching image digest", "digest", artifact.ImageDigest, "err", err) + return err + } + for _, item := range imageScanResult { + cveStores = append(cveStores, &item.CveStore) + } + env, err := impl.envRepository.FindById(pipeline.EnvironmentId) + if err != nil { + impl.logger.Errorw("error while fetching env", "err", err) + return err + } + blockCveList, err := impl.cvePolicyRepository.GetBlockedCVEList(cveStores, env.ClusterId, pipeline.EnvironmentId, pipeline.AppId, false) + if err != nil { + impl.logger.Errorw("error while fetching blocked cve list", "err", err) + return err + } + if len(blockCveList) > 0 { + isVulnerable = true + } + } + if isVulnerable == true { + if err = impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(runner, errors.New(pipelineConfig.FOUND_VULNERABILITY), triggeredBy); err != nil { + impl.logger.Errorw("error while updating current runner status to failed, TriggerDeployment", "wfrId", runner.Id, "err", err) + } + return nil + } + + releaseErr := impl.TriggerCD(artifact, cdWf.Id, savedWfr.Id, pipeline, triggeredAt) + // if releaseErr found, then the mark current deployment Failed and return + if releaseErr != nil { + err := impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(runner, releaseErr, triggeredBy) + if err != nil { + impl.logger.Errorw("error while updating current runner status to failed, updatePreviousDeploymentStatus", "cdWfr", runner.Id, "err", err) + } + return releaseErr + } + //skip updatePreviousDeploymentStatus if Async Install is enabled; handled inside SubscribeDevtronAsyncHelmInstallRequest + if !impl.IsDevtronAsyncInstallModeEnabled(pipeline.DeploymentAppType) { + err1 := impl.cdWorkflowCommonService.UpdatePreviousDeploymentStatus(runner, pipeline.Id, triggeredAt, triggeredBy) + if err1 != nil { + impl.logger.Errorw("error while update previous cd workflow runners", "err", err, "runner", runner, "pipelineId", pipeline.Id) + return err1 + } + } + return nil +} + +func (impl *TriggerServiceImpl) TriggerCD(artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, triggeredAt time.Time) error { + impl.logger.Debugw("automatic pipeline trigger attempt async", "artifactId", artifact.Id) + + return impl.triggerReleaseAsync(artifact, cdWorkflowId, wfrId, pipeline, triggeredAt) +} + +func (impl *TriggerServiceImpl) triggerReleaseAsync(artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, triggeredAt time.Time) error { + err := impl.validateAndTrigger(pipeline, artifact, cdWorkflowId, wfrId, triggeredAt) + if err != nil { + impl.logger.Errorw("error in trigger for pipeline", "pipelineId", strconv.Itoa(pipeline.Id)) + } + impl.logger.Debugw("trigger attempted for all pipeline ", "artifactId", artifact.Id) + return err +} + +func (impl *TriggerServiceImpl) validateAndTrigger(p *pipelineConfig.Pipeline, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, triggeredAt time.Time) error { + //TODO: verify this logicc + object := impl.enforcerUtil.GetAppRBACNameByAppId(p.AppId) + envApp := strings.Split(object, "/") + if len(envApp) != 2 { + impl.logger.Error("invalid req, app and env not found from rbac") + return errors.New("invalid req, app and env not found from rbac") + } + err := impl.releasePipeline(p, artifact, cdWorkflowId, wfrId, triggeredAt) + return err +} + +func (impl *TriggerServiceImpl) releasePipeline(pipeline *pipelineConfig.Pipeline, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, triggeredAt time.Time) error { + impl.logger.Debugw("triggering release for ", "cdPipelineId", pipeline.Id, "artifactId", artifact.Id) + + pipeline, err := impl.pipelineRepository.FindById(pipeline.Id) + if err != nil { + impl.logger.Errorw("error in fetching pipeline by pipelineId", "err", err) + return err + } + + request := &bean3.ValuesOverrideRequest{ + PipelineId: pipeline.Id, + UserId: artifact.CreatedBy, + CiArtifactId: artifact.Id, + AppId: pipeline.AppId, + CdWorkflowId: cdWorkflowId, + ForceTrigger: true, + DeploymentWithConfig: bean3.DEPLOYMENT_CONFIG_TYPE_LAST_SAVED, + WfrId: wfrId, + } + SetPipelineFieldsInOverrideRequest(request, pipeline) + + ctx, err := impl.argoUserService.BuildACDContext() + if err != nil { + impl.logger.Errorw("error in creating acd sync context", "pipelineId", pipeline.Id, "artifactId", artifact.Id, "err", err) + return err + } + //setting deployedBy as 1(system user) since case of auto trigger + id, _, err := impl.HandleCDTriggerRelease(request, ctx, triggeredAt, 1) + if err != nil { + impl.logger.Errorw("error in auto cd pipeline trigger", "pipelineId", pipeline.Id, "artifactId", artifact.Id, "err", err) + } else { + impl.logger.Infow("pipeline successfully triggered ", "cdPipelineId", pipeline.Id, "artifactId", artifact.Id, "releaseId", id) + } + return err +} + +func SetPipelineFieldsInOverrideRequest(overrideRequest *bean3.ValuesOverrideRequest, pipeline *pipelineConfig.Pipeline) { + overrideRequest.PipelineId = pipeline.Id + overrideRequest.PipelineName = pipeline.Name + overrideRequest.EnvId = pipeline.EnvironmentId + overrideRequest.EnvName = pipeline.Environment.Name + overrideRequest.ClusterId = pipeline.Environment.ClusterId + overrideRequest.AppId = pipeline.AppId + overrideRequest.AppName = pipeline.App.AppName + overrideRequest.DeploymentAppType = pipeline.DeploymentAppType +} + +func (impl *TriggerServiceImpl) HandleCDTriggerRelease(overrideRequest *bean3.ValuesOverrideRequest, ctx context.Context, + triggeredAt time.Time, deployedBy int32) (releaseNo int, manifest []byte, err error) { + if impl.IsDevtronAsyncInstallModeEnabled(overrideRequest.DeploymentAppType) { + // asynchronous mode of installation starts + return impl.workflowEventPublishService.TriggerHelmAsyncRelease(overrideRequest, ctx, triggeredAt, deployedBy) + } + // synchronous mode of installation starts + + valuesOverrideResponse, builtChartPath, err := impl.manifestCreationService.BuildManifestForTrigger(overrideRequest, triggeredAt, ctx) + _, span := otel.Tracer("orchestrator").Start(ctx, "CreateHistoriesForDeploymentTrigger") + err1 := impl.deployedConfigurationHistoryService.CreateHistoriesForDeploymentTrigger(valuesOverrideResponse.Pipeline, valuesOverrideResponse.PipelineStrategy, valuesOverrideResponse.EnvOverride, triggeredAt, deployedBy) + if err1 != nil { + impl.logger.Errorw("error in saving histories for trigger", "err", err1, "pipelineId", valuesOverrideResponse.Pipeline.Id, "wfrId", overrideRequest.WfrId) + } + span.End() + if err != nil { + impl.logger.Errorw("error in building merged manifest for trigger", "err", err) + return releaseNo, manifest, err + } + return impl.TriggerRelease(overrideRequest, valuesOverrideResponse, builtChartPath, ctx, triggeredAt, deployedBy) +} + // TriggerRelease will trigger Install/Upgrade request for Devtron App releases synchronously func (impl *TriggerServiceImpl) TriggerRelease(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, ctx context.Context, triggeredAt time.Time, triggeredBy int32) (releaseNo int, manifest []byte, err error) { @@ -665,3 +1245,19 @@ func (impl *TriggerServiceImpl) IsDevtronAsyncInstallModeEnabled(deploymentAppTy return impl.globalEnvVariables.EnableAsyncInstallDevtronChart && deploymentAppType == bean.Helm } + +func (impl *TriggerServiceImpl) deleteCorruptedPipelineStage(pipelineStage *repository.PipelineStage, triggeredBy int32) (error, bool) { + if pipelineStage != nil { + stageReq := &bean8.PipelineStageDto{ + Id: pipelineStage.Id, + Type: pipelineStage.Type, + } + err, deleted := impl.pipelineStageService.DeletePipelineStageIfReq(stageReq, triggeredBy) + if err != nil { + impl.logger.Errorw("error in deleting the corrupted pipeline stage", "err", err, "pipelineStageReq", stageReq) + return err, false + } + return nil, deleted + } + return nil, false +} diff --git a/pkg/eventProcessor/CentralEventProcessorService.go b/pkg/eventProcessor/CentralEventProcessorService.go index 5fa258499c..af65c90c38 100644 --- a/pkg/eventProcessor/CentralEventProcessorService.go +++ b/pkg/eventProcessor/CentralEventProcessorService.go @@ -50,5 +50,10 @@ func (impl *CentralEventProcessor) SubscribeAll() error { impl.logger.Errorw("error, SubscribeCDWorkflowStatusUpdate", "err", err) return err } + err = impl.workflowEventProcessor.SubscribeCICompleteEvent() + if err != nil { + impl.logger.Errorw("error, SubscribeCICompleteEvent", "err", err) + return err + } return nil } diff --git a/pkg/eventProcessor/bean/workflowEventBean.go b/pkg/eventProcessor/bean/workflowEventBean.go index 831cea2f2b..610fb33ba9 100644 --- a/pkg/eventProcessor/bean/workflowEventBean.go +++ b/pkg/eventProcessor/bean/workflowEventBean.go @@ -1,9 +1,11 @@ package bean import ( + "github.com/aws/aws-sdk-go-v2/service/ecr/types" "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/util" "time" ) @@ -25,3 +27,27 @@ type AsyncCdDeployEvent struct { TriggeredAt time.Time `json:"triggeredAt"` TriggeredBy int32 `json:"triggeredBy"` } + +type ImageDetailsFromCR struct { + ImageDetails []types.ImageDetail `json:"imageDetails"` + Region string `json:"region"` +} + +type CiCompleteEvent struct { + CiProjectDetails []bean3.CiProjectDetails `json:"ciProjectDetails"` + DockerImage string `json:"dockerImage" validate:"required,image-validator"` + Digest string `json:"digest"` + PipelineId int `json:"pipelineId"` + WorkflowId *int `json:"workflowId"` + TriggeredBy int32 `json:"triggeredBy"` + PipelineName string `json:"pipelineName"` + DataSource string `json:"dataSource"` + MaterialType string `json:"materialType"` + Metrics util.CIMetrics `json:"metrics"` + AppName string `json:"appName"` + IsArtifactUploaded bool `json:"isArtifactUploaded"` + FailureReason string `json:"failureReason"` + ImageDetailsFromCR *ImageDetailsFromCR `json:"imageDetailsFromCR"` + PluginRegistryArtifactDetails map[string][]string `json:"PluginRegistryArtifactDetails"` + PluginArtifactStage string `json:"pluginArtifactStage"` +} diff --git a/pkg/eventProcessor/in/WorkflowEventProcessorService.go b/pkg/eventProcessor/in/WorkflowEventProcessorService.go index 7faf6ea1a6..c17c3433e6 100644 --- a/pkg/eventProcessor/in/WorkflowEventProcessorService.go +++ b/pkg/eventProcessor/in/WorkflowEventProcessorService.go @@ -2,7 +2,9 @@ package in import ( "encoding/json" + "fmt" "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" + "github.com/aws/aws-sdk-go-v2/service/ecr/types" pubsub "github.com/devtron-labs/common-lib/pubsub-lib" "github.com/devtron-labs/common-lib/pubsub-lib/model" bean2 "github.com/devtron-labs/devtron/api/bean" @@ -10,17 +12,27 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" bean4 "github.com/devtron-labs/devtron/pkg/auth/user/bean" + "github.com/devtron-labs/devtron/pkg/deployment/deployedApp" + bean6 "github.com/devtron-labs/devtron/pkg/deployment/deployedApp/bean" + "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps" + bean5 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" + bean7 "github.com/devtron-labs/devtron/pkg/eventProcessor/out/bean" "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/workflow/cd" "github.com/devtron-labs/devtron/pkg/workflow/cd/adapter" bean3 "github.com/devtron-labs/devtron/pkg/workflow/cd/bean" + "github.com/devtron-labs/devtron/pkg/workflow/dag" + bean8 "github.com/devtron-labs/devtron/pkg/workflow/dag/bean" + util2 "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/argo" util "github.com/devtron-labs/devtron/util/event" "go.uber.org/zap" + "gopkg.in/go-playground/validator.v9" "k8s.io/utils/pointer" "strconv" + "time" ) type WorkflowEventProcessorImpl struct { @@ -28,13 +40,19 @@ type WorkflowEventProcessorImpl struct { pubSubClient *pubsub.PubSubClientServiceImpl cdWorkflowService cd.CdWorkflowService cdWorkflowRunnerService cd.CdWorkflowRunnerService - workflowDagExecutor pipeline.WorkflowDagExecutor + workflowDagExecutor dag.WorkflowDagExecutor argoUserService argo.ArgoUserService ciHandler pipeline.CiHandler cdHandler pipeline.CdHandler eventFactory client.EventFactory eventClient client.EventClient - //repositories import, to be removed + cdTriggerService devtronApps.TriggerService + deployedAppService deployedApp.DeployedAppService + webhookService pipeline.WebhookService + validator *validator.Validate + globalEnvVariables *util2.GlobalEnvVariables + cdWorkflowCommonService cd.CdWorkflowCommonService + //repositories import to be removed pipelineRepository pipelineConfig.PipelineRepository ciArtifactRepository repository.CiArtifactRepository cdWorkflowRepository pipelineConfig.CdWorkflowRepository @@ -44,10 +62,16 @@ func NewWorkflowEventProcessorImpl(logger *zap.SugaredLogger, pubSubClient *pubsub.PubSubClientServiceImpl, cdWorkflowService cd.CdWorkflowService, cdWorkflowRunnerService cd.CdWorkflowRunnerService, - workflowDagExecutor pipeline.WorkflowDagExecutor, + workflowDagExecutor dag.WorkflowDagExecutor, argoUserService argo.ArgoUserService, ciHandler pipeline.CiHandler, cdHandler pipeline.CdHandler, eventFactory client.EventFactory, eventClient client.EventClient, + cdTriggerService devtronApps.TriggerService, + deployedAppService deployedApp.DeployedAppService, + webhookService pipeline.WebhookService, + validator *validator.Validate, + globalEnvVariables *util2.GlobalEnvVariables, + cdWorkflowCommonService cd.CdWorkflowCommonService, pipelineRepository pipelineConfig.PipelineRepository, ciArtifactRepository repository.CiArtifactRepository, cdWorkflowRepository pipelineConfig.CdWorkflowRepository) (*WorkflowEventProcessorImpl, error) { @@ -62,6 +86,12 @@ func NewWorkflowEventProcessorImpl(logger *zap.SugaredLogger, eventFactory: eventFactory, eventClient: eventClient, workflowDagExecutor: workflowDagExecutor, + cdTriggerService: cdTriggerService, + deployedAppService: deployedAppService, + webhookService: webhookService, + validator: validator, + globalEnvVariables: globalEnvVariables, + cdWorkflowCommonService: cdWorkflowCommonService, pipelineRepository: pipelineRepository, ciArtifactRepository: ciArtifactRepository, cdWorkflowRepository: cdWorkflowRepository, @@ -82,7 +112,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDStageCompleteEvent() error { impl.logger.Errorw("could not get wf runner", "err", err) return } - triggerContext := pipeline.TriggerContext{ + triggerContext := bean5.TriggerContext{ ReferenceId: pointer.String(msg.MsgId), } if wfr.WorkflowType == bean2.CD_WORKFLOW_TYPE_PRE { @@ -112,7 +142,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDStageCompleteEvent() error { return "got message for cd stage completion", []interface{}{"workflowRunnerId", cdStageCompleteEvent.WorkflowRunnerId, "workflowId", cdStageCompleteEvent.WorkflowId, "cdPipelineId", cdStageCompleteEvent.CdPipelineId} } - validations := impl.workflowDagExecutor.GetTriggerValidateFuncs() + validations := impl.cdWorkflowCommonService.GetTriggerValidateFuncs() err := impl.pubSubClient.Subscribe(pubsub.CD_STAGE_COMPLETE_TOPIC, callback, loggerFunc, validations...) if err != nil { @@ -181,11 +211,11 @@ func (impl *WorkflowEventProcessorImpl) SubscribeTriggerBulkAction() error { impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", artifact.Id) } } - triggerContext := pipeline.TriggerContext{ + triggerContext := bean5.TriggerContext{ ReferenceId: pointer.String(msg.MsgId), } - triggerRequest := pipeline.TriggerRequest{ + triggerRequest := bean5.TriggerRequest{ CdWf: adapter.ConvertCdWorkflowDtoToDbObj(wf), //TODO: update object from db to dto Artifact: artifact, Pipeline: pipelineObj, @@ -193,7 +223,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeTriggerBulkAction() error { ApplyAuth: false, TriggerContext: triggerContext, } - err = impl.workflowDagExecutor.TriggerStageForBulk(triggerRequest) + err = impl.cdTriggerService.TriggerStageForBulk(triggerRequest) if err != nil { impl.logger.Errorw("error in cd trigger ", "err", err) wf.WorkflowStatus = pipelineConfig.TRIGGER_ERROR @@ -216,35 +246,32 @@ func (impl *WorkflowEventProcessorImpl) SubscribeTriggerBulkAction() error { return "got message for bulk deploy", []interface{}{"cdWorkflowId", cdWorkflow.Id} } - validations := impl.workflowDagExecutor.GetTriggerValidateFuncs() + validations := impl.cdWorkflowCommonService.GetTriggerValidateFuncs() return impl.pubSubClient.Subscribe(pubsub.BULK_DEPLOY_TOPIC, callback, loggerFunc, validations...) } func (impl *WorkflowEventProcessorImpl) SubscribeHibernateBulkAction() error { callback := func(msg *model.PubSubMsg) { - deploymentGroupAppWithEnv := new(pipeline.DeploymentGroupAppWithEnv) + deploymentGroupAppWithEnv := new(bean7.DeploymentGroupAppWithEnv) err := json.Unmarshal([]byte(msg.Data), deploymentGroupAppWithEnv) if err != nil { impl.logger.Error("Error while unmarshalling deploymentGroupAppWithEnv json object", err) return } - stopAppRequest := &pipeline.StopAppRequest{ + stopAppRequest := &bean6.StopAppRequest{ AppId: deploymentGroupAppWithEnv.AppId, EnvironmentId: deploymentGroupAppWithEnv.EnvironmentId, UserId: deploymentGroupAppWithEnv.UserId, RequestType: deploymentGroupAppWithEnv.RequestType, + ReferenceId: pointer.String(msg.MsgId), } ctx, err := impl.argoUserService.BuildACDContext() if err != nil { impl.logger.Errorw("error in creating acd sync context", "err", err) return } - triggerContext := pipeline.TriggerContext{ - ReferenceId: pointer.String(msg.MsgId), - Context: ctx, - } - _, err = impl.workflowDagExecutor.StopStartApp(triggerContext, stopAppRequest) + _, err = impl.deployedAppService.StopStartApp(ctx, stopAppRequest) if err != nil { impl.logger.Errorw("error in stop app request", "err", err) return @@ -253,7 +280,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeHibernateBulkAction() error { // add required logging here var loggerFunc pubsub.LoggerFunc = func(msg model.PubSubMsg) (string, []interface{}) { - deploymentGroupAppWithEnv := new(pipeline.DeploymentGroupAppWithEnv) + deploymentGroupAppWithEnv := new(bean7.DeploymentGroupAppWithEnv) err := json.Unmarshal([]byte(msg.Data), deploymentGroupAppWithEnv) if err != nil { return "error while unmarshalling deploymentGroupAppWithEnv json object", []interface{}{"err", err} @@ -347,7 +374,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDWorkflowStatusUpdate() error } if wfr != nil && executors.CheckIfReTriggerRequired(wfrStatus, wfStatus.Message, wfr.Status) { - err = impl.cdHandler.HandleCdStageReTrigger(wfr) + err = impl.workflowDagExecutor.HandleCdStageReTrigger(wfr) if err != nil { //check if this log required or not impl.logger.Errorw("error in HandleCdStageReTrigger", "error", err) @@ -384,3 +411,187 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDWorkflowStatusUpdate() error } return nil } + +func (impl *WorkflowEventProcessorImpl) SubscribeCICompleteEvent() error { + callback := func(msg *model.PubSubMsg) { + ciCompleteEvent := bean.CiCompleteEvent{} + err := json.Unmarshal([]byte(msg.Data), &ciCompleteEvent) + if err != nil { + impl.logger.Error("error while unmarshalling json data", "error", err) + return + } + impl.logger.Debugw("ci complete event for ci", "ciPipelineId", ciCompleteEvent.PipelineId) + req, err := impl.BuildCiArtifactRequest(ciCompleteEvent) + if err != nil { + return + } + + triggerContext := bean5.TriggerContext{ + ReferenceId: pointer.String(msg.MsgId), + } + + if ciCompleteEvent.FailureReason != "" { + req.FailureReason = ciCompleteEvent.FailureReason + err := impl.workflowDagExecutor.HandleCiStepFailedEvent(ciCompleteEvent.PipelineId, req) + if err != nil { + impl.logger.Error("Error while sending event for CI failure for pipelineID: ", + ciCompleteEvent.PipelineId, "request: ", req, "error: ", err) + return + } + } else if ciCompleteEvent.ImageDetailsFromCR != nil { + if len(ciCompleteEvent.ImageDetailsFromCR.ImageDetails) > 0 { + imageDetails := util2.GetReverseSortedImageDetails(ciCompleteEvent.ImageDetailsFromCR.ImageDetails) + digestWorkflowMap, err := impl.webhookService.HandleMultipleImagesFromEvent(imageDetails, *ciCompleteEvent.WorkflowId) + if err != nil { + impl.logger.Errorw("error in getting digest workflow map", "err", err, "workflowId", ciCompleteEvent.WorkflowId) + return + } + for _, detail := range imageDetails { + request, err := impl.BuildCIArtifactRequestForImageFromCR(detail, ciCompleteEvent.ImageDetailsFromCR.Region, ciCompleteEvent, digestWorkflowMap[*detail.ImageDigest].Id) + if err != nil { + impl.logger.Error("Error while creating request for pipelineID", "pipelineId", ciCompleteEvent.PipelineId, "err", err) + return + } + resp, err := impl.ValidateAndHandleCiSuccessEvent(triggerContext, ciCompleteEvent.PipelineId, request, detail.ImagePushedAt) + if err != nil { + return + } + impl.logger.Debug("response of handle ci success event for multiple images from plugin", "resp", resp) + } + } + + } else { + util2.TriggerCIMetrics(ciCompleteEvent.Metrics, impl.globalEnvVariables.ExposeCiMetrics, ciCompleteEvent.PipelineName, ciCompleteEvent.AppName) + resp, err := impl.ValidateAndHandleCiSuccessEvent(triggerContext, ciCompleteEvent.PipelineId, req, &time.Time{}) + if err != nil { + return + } + impl.logger.Debug(resp) + } + } + + // add required logging here + var loggerFunc pubsub.LoggerFunc = func(msg model.PubSubMsg) (string, []interface{}) { + ciCompleteEvent := bean.CiCompleteEvent{} + err := json.Unmarshal([]byte(msg.Data), &ciCompleteEvent) + if err != nil { + return "error while unmarshalling json data", []interface{}{"error", err} + } + return "got message for ci-completion", []interface{}{"ciPipelineId", ciCompleteEvent.PipelineId, "workflowId", ciCompleteEvent.WorkflowId} + } + + validations := impl.webhookService.GetTriggerValidateFuncs() + err := impl.pubSubClient.Subscribe(pubsub.CI_COMPLETE_TOPIC, callback, loggerFunc, validations...) + if err != nil { + impl.logger.Error(err) + return err + } + return nil +} + +func (impl *WorkflowEventProcessorImpl) ValidateAndHandleCiSuccessEvent(triggerContext bean5.TriggerContext, ciPipelineId int, request *bean8.CiArtifactWebhookRequest, imagePushedAt *time.Time) (int, error) { + validationErr := impl.validator.Struct(request) + if validationErr != nil { + impl.logger.Errorw("validation err, HandleCiSuccessEvent", "err", validationErr, "payload", request) + return 0, validationErr + } + buildArtifactId, err := impl.workflowDagExecutor.HandleCiSuccessEvent(triggerContext, ciPipelineId, request, imagePushedAt) + if err != nil { + impl.logger.Error("Error while sending event for CI success for pipelineID", + ciPipelineId, "request", request, "error", err) + return 0, err + } + return buildArtifactId, nil +} + +func (impl *WorkflowEventProcessorImpl) BuildCiArtifactRequest(event bean.CiCompleteEvent) (*bean8.CiArtifactWebhookRequest, error) { + var ciMaterialInfos []repository.CiMaterialInfo + for _, p := range event.CiProjectDetails { + var modifications []repository.Modification + + var branch string + var tag string + var webhookData repository.WebhookData + if p.SourceType == pipelineConfig.SOURCE_TYPE_BRANCH_FIXED { + branch = p.SourceValue + } else if p.SourceType == pipelineConfig.SOURCE_TYPE_WEBHOOK { + webhookData = repository.WebhookData{ + Id: p.WebhookData.Id, + EventActionType: p.WebhookData.EventActionType, + Data: p.WebhookData.Data, + } + } + + modification := repository.Modification{ + Revision: p.CommitHash, + ModifiedTime: p.CommitTime, + Author: p.Author, + Branch: branch, + Tag: tag, + WebhookData: webhookData, + Message: p.Message, + } + + modifications = append(modifications, modification) + ciMaterialInfo := repository.CiMaterialInfo{ + Material: repository.Material{ + GitConfiguration: repository.GitConfiguration{ + URL: p.GitRepository, + }, + Type: event.MaterialType, + }, + Changed: true, + Modifications: modifications, + } + ciMaterialInfos = append(ciMaterialInfos, ciMaterialInfo) + } + + materialBytes, err := json.Marshal(ciMaterialInfos) + if err != nil { + impl.logger.Errorw("cannot build ci artifact req", "err", err) + return nil, err + } + rawMaterialInfo := json.RawMessage(materialBytes) + fmt.Printf("Raw Message : %s\n", rawMaterialInfo) + + if event.TriggeredBy == 0 { + event.TriggeredBy = 1 // system triggered event + } + + request := &bean8.CiArtifactWebhookRequest{ + Image: event.DockerImage, + ImageDigest: event.Digest, + DataSource: event.DataSource, + PipelineName: event.PipelineName, + MaterialInfo: rawMaterialInfo, + UserId: event.TriggeredBy, + WorkflowId: event.WorkflowId, + IsArtifactUploaded: event.IsArtifactUploaded, + PluginRegistryArtifactDetails: event.PluginRegistryArtifactDetails, + PluginArtifactStage: event.PluginArtifactStage, + } + // if DataSource is empty, repository.WEBHOOK is considered as default + if request.DataSource == "" { + request.DataSource = repository.WEBHOOK + } + return request, nil +} + +func (impl *WorkflowEventProcessorImpl) BuildCIArtifactRequestForImageFromCR(imageDetails types.ImageDetail, region string, event bean.CiCompleteEvent, workflowId int) (*bean8.CiArtifactWebhookRequest, error) { + if event.TriggeredBy == 0 { + event.TriggeredBy = 1 // system triggered event + } + request := &bean8.CiArtifactWebhookRequest{ + Image: util2.ExtractEcrImage(*imageDetails.RegistryId, region, *imageDetails.RepositoryName, imageDetails.ImageTags[0]), + ImageDigest: *imageDetails.ImageDigest, + DataSource: event.DataSource, + PipelineName: event.PipelineName, + UserId: event.TriggeredBy, + WorkflowId: &workflowId, + IsArtifactUploaded: event.IsArtifactUploaded, + } + if request.DataSource == "" { + request.DataSource = repository.WEBHOOK + } + return request, nil +} diff --git a/pkg/eventProcessor/out/WorkflowEventPublishService.go b/pkg/eventProcessor/out/WorkflowEventPublishService.go index 834ac23554..95e27f8983 100644 --- a/pkg/eventProcessor/out/WorkflowEventPublishService.go +++ b/pkg/eventProcessor/out/WorkflowEventPublishService.go @@ -5,6 +5,7 @@ import ( "encoding/json" pubsub "github.com/devtron-labs/common-lib/pubsub-lib" bean3 "github.com/devtron-labs/devtron/api/bean" + "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/pkg/app/status" "github.com/devtron-labs/devtron/pkg/deployment/manifest" @@ -21,6 +22,7 @@ import ( ) type WorkflowEventPublishService interface { + TriggerBulkHibernateAsync(request bean.StopDeploymentGroupRequest) (interface{}, error) TriggerHelmAsyncRelease(overrideRequest *bean3.ValuesOverrideRequest, ctx context.Context, triggeredAt time.Time, triggeredBy int32) (releaseNo int, manifest []byte, err error) TriggerBulkDeploymentAsync(requests []*bean.BulkTriggerRequest, UserId int32) (interface{}, error) @@ -37,6 +39,7 @@ type WorkflowEventPublishServiceImpl struct { cdWorkflowRepository pipelineConfig.CdWorkflowRepository pipelineRepository pipelineConfig.PipelineRepository + groupRepository repository.DeploymentGroupRepository } func NewWorkflowEventPublishServiceImpl(logger *zap.SugaredLogger, @@ -47,7 +50,8 @@ func NewWorkflowEventPublishServiceImpl(logger *zap.SugaredLogger, pipelineStatusTimelineService status.PipelineStatusTimelineService, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, - pipelineRepository pipelineConfig.PipelineRepository) (*WorkflowEventPublishServiceImpl, error) { + pipelineRepository pipelineConfig.PipelineRepository, + groupRepository repository.DeploymentGroupRepository) (*WorkflowEventPublishServiceImpl, error) { config, err := types.GetCdConfig() if err != nil { return nil, err @@ -63,10 +67,41 @@ func NewWorkflowEventPublishServiceImpl(logger *zap.SugaredLogger, cdWorkflowRepository: cdWorkflowRepository, pipelineRepository: pipelineRepository, + groupRepository: groupRepository, } return impl, nil } +func (impl *WorkflowEventPublishServiceImpl) TriggerBulkHibernateAsync(request bean.StopDeploymentGroupRequest) (interface{}, error) { + dg, err := impl.groupRepository.FindByIdWithApp(request.DeploymentGroupId) + if err != nil { + impl.logger.Errorw("error while fetching dg", "err", err) + return nil, err + } + + for _, app := range dg.DeploymentGroupApps { + deploymentGroupAppWithEnv := &bean.DeploymentGroupAppWithEnv{ + AppId: app.AppId, + EnvironmentId: dg.EnvironmentId, + DeploymentGroupId: dg.Id, + Active: dg.Active, + UserId: request.UserId, + RequestType: request.RequestType, + } + + data, err := json.Marshal(deploymentGroupAppWithEnv) + if err != nil { + impl.logger.Errorw("error while writing app stop event to nats ", "app", app.AppId, "deploymentGroup", app.DeploymentGroupId, "err", err) + } else { + err = impl.pubSubClient.Publish(pubsub.BULK_HIBERNATE_TOPIC, string(data)) + if err != nil { + impl.logger.Errorw("Error while publishing request", "topic", pubsub.BULK_HIBERNATE_TOPIC, "error", err) + } + } + } + return nil, nil +} + // TriggerHelmAsyncRelease will publish async helm Install/Upgrade request event for Devtron App releases func (impl *WorkflowEventPublishServiceImpl) TriggerHelmAsyncRelease(overrideRequest *bean3.ValuesOverrideRequest, ctx context.Context, triggeredAt time.Time, triggeredBy int32) (releaseNo int, manifest []byte, err error) { // build merged values and save PCO history for the release diff --git a/pkg/eventProcessor/out/bean/bean.go b/pkg/eventProcessor/out/bean/bean.go index 974c99319b..cb36a44bb1 100644 --- a/pkg/eventProcessor/out/bean/bean.go +++ b/pkg/eventProcessor/out/bean/bean.go @@ -1,6 +1,23 @@ package bean +import bean4 "github.com/devtron-labs/devtron/pkg/deployment/deployedApp/bean" + type BulkTriggerRequest struct { CiArtifactId int `sql:"ci_artifact_id"` PipelineId int `sql:"pipeline_id"` } + +type StopDeploymentGroupRequest struct { + DeploymentGroupId int `json:"deploymentGroupId" validate:"required"` + UserId int32 `json:"userId"` + RequestType bean4.RequestType `json:"requestType" validate:"oneof=START STOP"` +} + +type DeploymentGroupAppWithEnv struct { + EnvironmentId int `json:"environmentId"` + DeploymentGroupId int `json:"deploymentGroupId"` + AppId int `json:"appId"` + Active bool `json:"active"` + UserId int32 `json:"userId"` + RequestType bean4.RequestType `json:"requestType" validate:"oneof=START STOP"` +} diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index b1e15cf2f8..856100c032 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -19,34 +19,21 @@ package pipeline import ( "bufio" - "context" "errors" "fmt" - client "github.com/devtron-labs/devtron/api/helm-app/service" "os" "path/filepath" "strconv" "strings" "time" - application2 "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" blob_storage "github.com/devtron-labs/common-lib/blob-storage" - pubub "github.com/devtron-labs/common-lib/pubsub-lib" "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/devtron/api/bean" - "github.com/devtron-labs/devtron/client/argocdServer" - "github.com/devtron-labs/devtron/client/argocdServer/application" - client2 "github.com/devtron-labs/devtron/client/events" "github.com/devtron-labs/devtron/internal/sql/repository" - app2 "github.com/devtron-labs/devtron/internal/sql/repository/app" - "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" - "github.com/devtron-labs/devtron/pkg/app" - "github.com/devtron-labs/devtron/pkg/app/status" - app_status "github.com/devtron-labs/devtron/pkg/appStatus" - repository3 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" "github.com/devtron-labs/devtron/pkg/auth/user" "github.com/devtron-labs/devtron/pkg/cluster" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" @@ -54,15 +41,12 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/types" resourceGroup2 "github.com/devtron-labs/devtron/pkg/resourceGroup" - "github.com/devtron-labs/devtron/pkg/sql" util3 "github.com/devtron-labs/devtron/util" - "github.com/devtron-labs/devtron/util/argo" "github.com/devtron-labs/devtron/util/rbac" "github.com/go-pg/pg" "go.opentelemetry.io/otel" "go.uber.org/zap" "k8s.io/client-go/rest" - "k8s.io/utils/strings/slices" ) const ( @@ -72,7 +56,6 @@ const ( ) type CdHandler interface { - HandleCdStageReTrigger(runner *pipelineConfig.CdWorkflowRunner) error UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus) (int, string, error) GetCdBuildHistory(appId int, environmentId int, pipelineId int, offset int, size int) ([]pipelineConfig.CdWorkflowWithArtifact, error) GetRunningWorkflowLogs(environmentId int, pipelineId int, workflowId int) (*bufio.Reader, func() error, error) @@ -81,98 +64,60 @@ type CdHandler interface { FetchCdPrePostStageStatus(pipelineId int) ([]pipelineConfig.CdWorkflowWithArtifact, error) CancelStage(workflowRunnerId int, userId int32) (int, error) FetchAppWorkflowStatusForTriggerView(appId int) ([]*pipelineConfig.CdWorkflowStatus, error) - CheckHelmAppStatusPeriodicallyAndUpdateInDb(helmPipelineStatusCheckEligibleTime int, getPipelineDeployedWithinHours int) error - CheckArgoAppStatusPeriodicallyAndUpdateInDb(getPipelineDeployedBeforeMinutes int, getPipelineDeployedWithinHours int) error - CheckArgoPipelineTimelineStatusPeriodicallyAndUpdateInDb(pendingSinceSeconds int, timeForDegradation int) error - UpdatePipelineTimelineAndStatusByLiveApplicationFetch(triggerContext TriggerContext, pipeline *pipelineConfig.Pipeline, installedApp repository3.InstalledApps, userId int32) (err error, isTimelineUpdated bool) - CheckAndSendArgoPipelineStatusSyncEventIfNeeded(pipelineId int, installedAppVersionId int, userId int32, isAppStoreApplication bool) FetchAppWorkflowStatusForTriggerViewForEnvironment(request resourceGroup2.ResourceGroupingRequest, token string) ([]*pipelineConfig.CdWorkflowStatus, error) FetchAppDeploymentStatusForEnvironments(request resourceGroup2.ResourceGroupingRequest, token string) ([]*pipelineConfig.AppDeploymentStatus, error) DeactivateImageReservationPathsOnFailure(imagePathReservationIds []int) error } type CdHandlerImpl struct { - Logger *zap.SugaredLogger - userService user.UserService - ciLogService CiLogService - ciArtifactRepository repository.CiArtifactRepository - ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository - cdWorkflowRepository pipelineConfig.CdWorkflowRepository - envRepository repository2.EnvironmentRepository - pipelineRepository pipelineConfig.PipelineRepository - ciWorkflowRepository pipelineConfig.CiWorkflowRepository - helmAppService client.HelmAppService - pipelineOverrideRepository chartConfig.PipelineOverrideRepository - workflowDagExecutor WorkflowDagExecutor - appListingService app.AppListingService - appListingRepository repository.AppListingRepository - pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository - application application.ServiceClient - argoUserService argo.ArgoUserService - deploymentEventHandler app.DeploymentEventHandler - eventClient client2.EventClient - pipelineStatusTimelineResourcesService status.PipelineStatusTimelineResourcesService - pipelineStatusSyncDetailService status.PipelineStatusSyncDetailService - pipelineStatusTimelineService status.PipelineStatusTimelineService - appService app.AppService - appStatusService app_status.AppStatusService - enforcerUtil rbac.EnforcerUtil - installedAppRepository repository3.InstalledAppRepository - installedAppVersionHistoryRepository repository3.InstalledAppVersionHistoryRepository - appRepository app2.AppRepository - resourceGroupService resourceGroup2.ResourceGroupService - imageTaggingService ImageTaggingService - k8sUtil *k8s.K8sServiceImpl - workflowService WorkflowService - config *types.CdConfig - clusterService cluster.ClusterService - blobConfigStorageService BlobStorageConfigService - customTagService CustomTagService - argocdClientWrapperService argocdServer.ArgoClientWrapperService - AppConfig *app.AppServiceConfig - acdConfig *argocdServer.ACDConfig + Logger *zap.SugaredLogger + userService user.UserService + ciLogService CiLogService + ciArtifactRepository repository.CiArtifactRepository + ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository + cdWorkflowRepository pipelineConfig.CdWorkflowRepository + envRepository repository2.EnvironmentRepository + pipelineRepository pipelineConfig.PipelineRepository + ciWorkflowRepository pipelineConfig.CiWorkflowRepository + enforcerUtil rbac.EnforcerUtil + resourceGroupService resourceGroup2.ResourceGroupService + imageTaggingService ImageTaggingService + k8sUtil *k8s.K8sServiceImpl + workflowService WorkflowService + config *types.CdConfig + clusterService cluster.ClusterService + blobConfigStorageService BlobStorageConfigService + customTagService CustomTagService } -func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, ciLogService CiLogService, ciArtifactRepository repository.CiArtifactRepository, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, pipelineRepository pipelineConfig.PipelineRepository, envRepository repository2.EnvironmentRepository, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, helmAppService client.HelmAppService, pipelineOverrideRepository chartConfig.PipelineOverrideRepository, workflowDagExecutor WorkflowDagExecutor, appListingService app.AppListingService, appListingRepository repository.AppListingRepository, pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository, application application.ServiceClient, argoUserService argo.ArgoUserService, deploymentEventHandler app.DeploymentEventHandler, eventClient client2.EventClient, pipelineStatusTimelineResourcesService status.PipelineStatusTimelineResourcesService, pipelineStatusSyncDetailService status.PipelineStatusSyncDetailService, pipelineStatusTimelineService status.PipelineStatusTimelineService, appService app.AppService, appStatusService app_status.AppStatusService, enforcerUtil rbac.EnforcerUtil, installedAppRepository repository3.InstalledAppRepository, installedAppVersionHistoryRepository repository3.InstalledAppVersionHistoryRepository, appRepository app2.AppRepository, resourceGroupService resourceGroup2.ResourceGroupService, imageTaggingService ImageTaggingService, k8sUtil *k8s.K8sServiceImpl, workflowService WorkflowService, clusterService cluster.ClusterService, blobConfigStorageService BlobStorageConfigService, customTagService CustomTagService, argocdClientWrapperService argocdServer.ArgoClientWrapperService, AppConfig *app.AppServiceConfig, acdConfig *argocdServer.ACDConfig) *CdHandlerImpl { +func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, + cdWorkflowRepository pipelineConfig.CdWorkflowRepository, ciLogService CiLogService, + ciArtifactRepository repository.CiArtifactRepository, + ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, + pipelineRepository pipelineConfig.PipelineRepository, envRepository repository2.EnvironmentRepository, + ciWorkflowRepository pipelineConfig.CiWorkflowRepository, enforcerUtil rbac.EnforcerUtil, + resourceGroupService resourceGroup2.ResourceGroupService, + imageTaggingService ImageTaggingService, k8sUtil *k8s.K8sServiceImpl, + workflowService WorkflowService, clusterService cluster.ClusterService, + blobConfigStorageService BlobStorageConfigService, customTagService CustomTagService) *CdHandlerImpl { cdh := &CdHandlerImpl{ - Logger: Logger, - userService: userService, - ciLogService: ciLogService, - cdWorkflowRepository: cdWorkflowRepository, - ciArtifactRepository: ciArtifactRepository, - ciPipelineMaterialRepository: ciPipelineMaterialRepository, - envRepository: envRepository, - pipelineRepository: pipelineRepository, - ciWorkflowRepository: ciWorkflowRepository, - helmAppService: helmAppService, - pipelineOverrideRepository: pipelineOverrideRepository, - workflowDagExecutor: workflowDagExecutor, - appListingService: appListingService, - appListingRepository: appListingRepository, - pipelineStatusTimelineRepository: pipelineStatusTimelineRepository, - application: application, - argoUserService: argoUserService, - deploymentEventHandler: deploymentEventHandler, - eventClient: eventClient, - pipelineStatusTimelineResourcesService: pipelineStatusTimelineResourcesService, - pipelineStatusSyncDetailService: pipelineStatusSyncDetailService, - pipelineStatusTimelineService: pipelineStatusTimelineService, - appService: appService, - appStatusService: appStatusService, - enforcerUtil: enforcerUtil, - installedAppRepository: installedAppRepository, - installedAppVersionHistoryRepository: installedAppVersionHistoryRepository, - appRepository: appRepository, - resourceGroupService: resourceGroupService, - imageTaggingService: imageTaggingService, - k8sUtil: k8sUtil, - workflowService: workflowService, - clusterService: clusterService, - blobConfigStorageService: blobConfigStorageService, - customTagService: customTagService, - argocdClientWrapperService: argocdClientWrapperService, - AppConfig: AppConfig, - acdConfig: acdConfig, + Logger: Logger, + userService: userService, + ciLogService: ciLogService, + cdWorkflowRepository: cdWorkflowRepository, + ciArtifactRepository: ciArtifactRepository, + ciPipelineMaterialRepository: ciPipelineMaterialRepository, + envRepository: envRepository, + pipelineRepository: pipelineRepository, + ciWorkflowRepository: ciWorkflowRepository, + enforcerUtil: enforcerUtil, + resourceGroupService: resourceGroupService, + imageTaggingService: imageTaggingService, + k8sUtil: k8sUtil, + workflowService: workflowService, + clusterService: clusterService, + blobConfigStorageService: blobConfigStorageService, + customTagService: customTagService, } config, err := types.GetCdConfig() if err != nil { @@ -182,432 +127,12 @@ func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, c return cdh } -type ArgoPipelineStatusSyncEvent struct { - PipelineId int `json:"pipelineId"` - InstalledAppVersionId int `json:"installedAppVersionId"` - UserId int32 `json:"userId"` - IsAppStoreApplication bool `json:"isAppStoreApplication"` -} - const NotTriggered string = "Not Triggered" const NotDeployed = "Not Deployed" const WorklowTypeDeploy = "DEPLOY" const WorklowTypePre = "PRE" const WorklowTypePost = "POST" -func (impl *CdHandlerImpl) HandleCdStageReTrigger(runner *pipelineConfig.CdWorkflowRunner) error { - // do not re-trigger if retries = 0 - if !impl.config.WorkflowRetriesEnabled() { - impl.Logger.Debugw("cd stage workflow re-triggering is not enabled") - return nil - } - - impl.Logger.Infow("re triggering cd stage ", "runnerId", runner.Id) - var err error - // add comment for this logic - if runner.RefCdWorkflowRunnerId != 0 { - runner, err = impl.cdWorkflowRepository.FindWorkflowRunnerById(runner.RefCdWorkflowRunnerId) - if err != nil { - impl.Logger.Errorw("error in FindWorkflowRunnerById by id ", "err", err, "wfrId", runner.RefCdWorkflowRunnerId) - return err - } - } - retryCnt, err := impl.cdWorkflowRepository.FindRetriedWorkflowCountByReferenceId(runner.Id) - if err != nil { - impl.Logger.Errorw("error in FindRetriedWorkflowCountByReferenceId ", "err", err, "cdWorkflowRunnerId", runner.Id) - return err - } - - if retryCnt >= impl.config.MaxCdWorkflowRunnerRetries { - impl.Logger.Infow("maximum retries for this workflow are exhausted, not re-triggering again", "retries", retryCnt, "wfrId", runner.Id) - return nil - } - - triggerRequest := TriggerRequest{ - CdWf: runner.CdWorkflow, - Pipeline: runner.CdWorkflow.Pipeline, - Artifact: runner.CdWorkflow.CiArtifact, - TriggeredBy: 1, - ApplyAuth: false, - RefCdWorkflowRunnerId: runner.Id, - TriggerContext: TriggerContext{ - Context: context.Background(), - }, - } - - if runner.WorkflowType == bean.CD_WORKFLOW_TYPE_PRE { - err = impl.workflowDagExecutor.TriggerPreStage(triggerRequest) - if err != nil { - impl.Logger.Errorw("error in TriggerPreStage ", "err", err, "cdWorkflowRunnerId", runner.Id) - return err - } - } else if runner.WorkflowType == bean.CD_WORKFLOW_TYPE_POST { - err = impl.workflowDagExecutor.TriggerPostStage(triggerRequest) - if err != nil { - impl.Logger.Errorw("error in TriggerPostStage ", "err", err, "cdWorkflowRunnerId", runner.Id) - return err - } - } - - impl.Logger.Infow("cd stage re triggered for runner", "runnerId", runner.Id) - return nil -} - -func (impl *CdHandlerImpl) CheckArgoAppStatusPeriodicallyAndUpdateInDb(getPipelineDeployedBeforeMinutes int, getPipelineDeployedWithinHours int) error { - pipelines, err := impl.pipelineRepository.GetArgoPipelinesHavingLatestTriggerStuckInNonTerminalStatuses(getPipelineDeployedBeforeMinutes, getPipelineDeployedWithinHours) - if err != nil { - impl.Logger.Errorw("error in getting pipelines having latest trigger stuck in non terminal statuses", "err", err) - return err - } - impl.Logger.Debugw("received stuck argo cd pipelines", "pipelines", pipelines, "number of pipelines", len(pipelines)) - - for _, pipeline := range pipelines { - impl.CheckAndSendArgoPipelineStatusSyncEventIfNeeded(pipeline.Id, 0, 1, false) - } - - installedAppVersions, err := impl.installedAppRepository.GetArgoPipelinesHavingLatestTriggerStuckInNonTerminalStatusesForAppStore(getPipelineDeployedBeforeMinutes, getPipelineDeployedWithinHours) - if err != nil { - impl.Logger.Errorw("error in getting installedAppVersions having latest trigger stuck in non terminal statuses", "err", err) - return err - } - impl.Logger.Debugw("received stuck argo installed appStore app", "installedAppVersions", installedAppVersions, "number of triggers", len(installedAppVersions)) - - for _, installedAppVersion := range installedAppVersions { - impl.CheckAndSendArgoPipelineStatusSyncEventIfNeeded(0, installedAppVersion.Id, 1, true) - } - return nil -} - -func (impl *CdHandlerImpl) CheckArgoPipelineTimelineStatusPeriodicallyAndUpdateInDb(pendingSinceSeconds int, timeForDegradation int) error { - // getting all the progressing status that are stuck since some time after kubectl apply success sync stage - // and are not eligible for CheckArgoAppStatusPeriodicallyAndUpdateInDb - pipelines, err := impl.pipelineRepository.GetArgoPipelinesHavingTriggersStuckInLastPossibleNonTerminalTimelines(pendingSinceSeconds, timeForDegradation) - if err != nil && err != pg.ErrNoRows { - impl.Logger.Errorw("err in GetArgoPipelinesHavingTriggersStuckInLastPossibleNonTerminalTimelines", "err", err) - return err - } - impl.Logger.Debugw("received argo cd pipelines stuck at kubectl apply synced stage", "pipelines", pipelines) - - installedAppVersions, err := impl.installedAppRepository.GetArgoPipelinesHavingTriggersStuckInLastPossibleNonTerminalTimelinesForAppStore(pendingSinceSeconds, timeForDegradation) - if err != nil && err != pg.ErrNoRows { - impl.Logger.Errorw("err in GetArgoPipelinesHavingTriggersStuckInLastPossibleNonTerminalTimelinesForAppStore", "err", err) - return err - } - - impl.Logger.Debugw("received argo appStore application stuck at kubectl apply synced stage", "pipelines", installedAppVersions) - for _, pipeline := range pipelines { - impl.CheckAndSendArgoPipelineStatusSyncEventIfNeeded(pipeline.Id, 0, 1, false) - } - - for _, installedAppVersion := range installedAppVersions { - impl.CheckAndSendArgoPipelineStatusSyncEventIfNeeded(0, installedAppVersion.Id, 1, true) - } - return nil -} - -func (impl *CdHandlerImpl) CheckAndSendArgoPipelineStatusSyncEventIfNeeded(pipelineId int, installedAppVersionId int, userId int32, isAppStoreApplication bool) { - var lastSyncTime time.Time - var err error - if isAppStoreApplication { - lastSyncTime, err = impl.pipelineStatusSyncDetailService.GetLastSyncTimeForLatestInstalledAppVersionHistoryByInstalledAppVersionId(installedAppVersionId) - } else { - lastSyncTime, err = impl.pipelineStatusSyncDetailService.GetLastSyncTimeForLatestCdWfrByCdPipelineId(pipelineId) - } - if err != nil { - impl.Logger.Errorw("error in getting last sync time by pipelineId", "err", err, "pipelineId", pipelineId, "installedAppVersionHistoryId", installedAppVersionId) - return - } - - // sync argocd app - if pipelineId != 0 { - err := impl.syncACDDevtronApps(impl.AppConfig.ArgocdManualSyncCronPipelineDeployedBefore, pipelineId) - if err != nil { - impl.Logger.Errorw("error in syncing devtron apps deployed via argoCD", "err", err) - return - } - } - if installedAppVersionId != 0 { - err := impl.SyncACDHelmApps(impl.AppConfig.ArgocdManualSyncCronPipelineDeployedBefore, installedAppVersionId) - if err != nil { - impl.Logger.Errorw("error in syncing Helm apps deployed via argoCD", "err", err) - return - } - } - - // pipelineId can be cdPipelineId or installedAppVersionId, using isAppStoreApplication flag to identify between them - if lastSyncTime.IsZero() || (!lastSyncTime.IsZero() && time.Since(lastSyncTime) > 5*time.Second) { // create new nats event - statusUpdateEvent := ArgoPipelineStatusSyncEvent{ - PipelineId: pipelineId, - InstalledAppVersionId: installedAppVersionId, - UserId: userId, - IsAppStoreApplication: isAppStoreApplication, - } - // write event - err = impl.eventClient.WriteNatsEvent(pubub.ARGO_PIPELINE_STATUS_UPDATE_TOPIC, statusUpdateEvent) - if err != nil { - impl.Logger.Errorw("error in writing nats event", "topic", pubub.ARGO_PIPELINE_STATUS_UPDATE_TOPIC, "payload", statusUpdateEvent) - } - } -} - -func (impl *CdHandlerImpl) UpdatePipelineTimelineAndStatusByLiveApplicationFetch(triggerContext TriggerContext, pipeline *pipelineConfig.Pipeline, installedApp repository3.InstalledApps, userId int32) (error, bool) { - isTimelineUpdated := false - isSucceeded := false - var pipelineOverride *chartConfig.PipelineOverride - if pipeline != nil { - isAppStore := false - cdWfr, err := impl.cdWorkflowRepository.FindLastStatusByPipelineIdAndRunnerType(pipeline.Id, bean.CD_WORKFLOW_TYPE_DEPLOY) - if err != nil { - impl.Logger.Errorw("error in getting latest cdWfr by cdPipelineId", "err", err, "pipelineId", pipeline.Id) - return nil, isTimelineUpdated - } - impl.Logger.Debugw("ARGO_PIPELINE_STATUS_UPDATE_REQ", "stage", "checkingDeploymentStatus", "argoAppName", pipeline, "cdWfr", cdWfr) - if util3.IsTerminalStatus(cdWfr.Status) { - // drop event - return nil, isTimelineUpdated - } - - if !impl.acdConfig.ArgoCDAutoSyncEnabled { - // if manual sync check for application sync status - isArgoAppSynced := impl.pipelineStatusTimelineService.GetArgoAppSyncStatus(cdWfr.Id) - if !isArgoAppSynced { - return nil, isTimelineUpdated - } - } - // this should only be called when we have git-ops configured - // try fetching status from argo cd - acdToken, err := impl.argoUserService.GetLatestDevtronArgoCdUserToken() - if err != nil { - impl.Logger.Errorw("error in getting acd token", "err", err) - } - ctx := context.WithValue(context.Background(), "token", acdToken) - query := &application2.ApplicationQuery{ - Name: &pipeline.DeploymentAppName, - } - app, err := impl.application.Get(ctx, query) - if err != nil { - impl.Logger.Errorw("error in getting acd application", "err", err, "argoAppName", pipeline) - // updating cdWfr status - cdWfr.Status = pipelineConfig.WorkflowUnableToFetchState - cdWfr.UpdatedOn = time.Now() - cdWfr.UpdatedBy = 1 - err = impl.cdWorkflowRepository.UpdateWorkFlowRunner(&cdWfr) - if err != nil { - impl.Logger.Errorw("error on update cd workflow runner", "cdWfr", cdWfr, "err", err) - return err, isTimelineUpdated - } - // creating cd pipeline status timeline - timeline := &pipelineConfig.PipelineStatusTimeline{ - CdWorkflowRunnerId: cdWfr.Id, - Status: pipelineConfig.TIMELINE_STATUS_UNABLE_TO_FETCH_STATUS, - StatusDetail: "Failed to connect to Argo CD to fetch deployment status.", - StatusTime: time.Now(), - AuditLog: sql.AuditLog{ - CreatedBy: userId, - CreatedOn: time.Now(), - UpdatedBy: userId, - UpdatedOn: time.Now(), - }, - } - err = impl.pipelineStatusTimelineService.SaveTimeline(timeline, nil, isAppStore) - if err != nil { - impl.Logger.Errorw("error in creating timeline status for app", "err", err, "timeline", timeline) - return err, isTimelineUpdated - } - } else { - if app == nil { - impl.Logger.Errorw("found empty argo application object", "appName", pipeline.DeploymentAppName) - return fmt.Errorf("found empty argo application object"), isTimelineUpdated - } - isSucceeded, isTimelineUpdated, pipelineOverride, err = impl.appService.UpdateDeploymentStatusForGitOpsPipelines(app, time.Now(), isAppStore) - if err != nil { - impl.Logger.Errorw("error in updating deployment status for gitOps cd pipelines", "app", app) - return err, isTimelineUpdated - } - appStatus := app.Status.Health.Status - err = impl.appStatusService.UpdateStatusWithAppIdEnvId(pipeline.AppId, pipeline.EnvironmentId, string(appStatus)) - if err != nil { - impl.Logger.Errorw("error occurred while updating app-status for cd pipeline", "err", err, "appId", pipeline.AppId, "envId", pipeline.EnvironmentId) - impl.Logger.Debugw("ignoring the error, UpdateStatusWithAppIdEnvId", "err", err, "appId", pipeline.AppId, "envId", pipeline.EnvironmentId) - } - } - if isSucceeded { - // handling deployment success event - err = impl.workflowDagExecutor.HandleDeploymentSuccessEvent(triggerContext, pipelineOverride) - if err != nil { - impl.Logger.Errorw("error in handling deployment success event", "pipelineOverride", pipelineOverride, "err", err) - return err, isTimelineUpdated - } - } - } else { - isAppStore := true - installedAppVersionHistory, err := impl.installedAppVersionHistoryRepository.GetLatestInstalledAppVersionHistoryByInstalledAppId(installedApp.Id) - if err != nil { - impl.Logger.Errorw("error in getting latest installedAppVersionHistory by installedAppId", "err", err, "installedAppId", installedApp.Id) - return nil, isTimelineUpdated - } - impl.Logger.Debugw("ARGO_PIPELINE_STATUS_UPDATE_REQ", "stage", "checkingDeploymentStatus", "argoAppName", installedApp, "installedAppVersionHistory", installedAppVersionHistory) - if util3.IsTerminalStatus(installedAppVersionHistory.Status) { - // drop event - return nil, isTimelineUpdated - } - if !impl.acdConfig.ArgoCDAutoSyncEnabled { - isArgoAppSynced := impl.pipelineStatusTimelineService.GetArgoAppSyncStatusForAppStore(installedAppVersionHistory.Id) - if !isArgoAppSynced { - return nil, isTimelineUpdated - } - } - appDetails, err := impl.appRepository.FindActiveById(installedApp.AppId) - if err != nil { - impl.Logger.Errorw("error in getting appDetails from appId", "err", err) - return nil, isTimelineUpdated - } - // TODO if Environment object in installedApp is nil then fetch envDetails also from envRepository - envDetail, err := impl.envRepository.FindById(installedApp.EnvironmentId) - if err != nil { - impl.Logger.Errorw("error in getting envDetails from environment id", "err", err) - return nil, isTimelineUpdated - } - var acdAppName string - if len(installedApp.Environment.Name) != 0 { - acdAppName = appDetails.AppName + installedApp.Environment.Name - } else { - acdAppName = appDetails.AppName + "-" + envDetail.Name - } - - // this should only be called when we have git-ops configured - // try fetching status from argo cd - acdToken, err := impl.argoUserService.GetLatestDevtronArgoCdUserToken() - if err != nil { - impl.Logger.Errorw("error in getting acd token", "err", err) - } - - ctx := context.WithValue(context.Background(), "token", acdToken) - query := &application2.ApplicationQuery{ - Name: &acdAppName, - } - app, err := impl.application.Get(ctx, query) - if err != nil { - impl.Logger.Errorw("error in getting acd application", "err", err, "argoAppName", installedApp) - // updating cdWfr status - installedAppVersionHistory.Status = pipelineConfig.WorkflowUnableToFetchState - installedAppVersionHistory.UpdatedOn = time.Now() - installedAppVersionHistory.UpdatedBy = 1 - installedAppVersionHistory, err = impl.installedAppVersionHistoryRepository.UpdateInstalledAppVersionHistory(installedAppVersionHistory, nil) - if err != nil { - impl.Logger.Errorw("error on update installedAppVersionHistory", "installedAppVersionHistory", installedAppVersionHistory, "err", err) - return err, isTimelineUpdated - } - // creating installedApp pipeline status timeline - timeline := &pipelineConfig.PipelineStatusTimeline{ - InstalledAppVersionHistoryId: installedAppVersionHistory.Id, - Status: pipelineConfig.TIMELINE_STATUS_UNABLE_TO_FETCH_STATUS, - StatusDetail: "Failed to connect to Argo CD to fetch deployment status.", - StatusTime: time.Now(), - AuditLog: sql.AuditLog{ - CreatedBy: userId, - CreatedOn: time.Now(), - UpdatedBy: userId, - UpdatedOn: time.Now(), - }, - } - err = impl.pipelineStatusTimelineService.SaveTimeline(timeline, nil, isAppStore) - if err != nil { - impl.Logger.Errorw("error in creating timeline status for app", "err", err, "timeline", timeline) - return err, isTimelineUpdated - } - } else { - if app == nil { - impl.Logger.Errorw("found empty argo application object", "appName", acdAppName) - return fmt.Errorf("found empty argo application object"), isTimelineUpdated - } - isSucceeded, isTimelineUpdated, pipelineOverride, err = impl.appService.UpdateDeploymentStatusForGitOpsPipelines(app, time.Now(), isAppStore) - if err != nil { - impl.Logger.Errorw("error in updating deployment status for gitOps cd pipelines", "app", app) - return err, isTimelineUpdated - } - appStatus := app.Status.Health.Status - err = impl.appStatusService.UpdateStatusWithAppIdEnvId(installedApp.AppId, installedApp.EnvironmentId, string(appStatus)) - if err != nil { - impl.Logger.Errorw("error occurred while updating app-status for installed app", "err", err, "appId", installedApp.AppId, "envId", installedApp.EnvironmentId) - impl.Logger.Debugw("ignoring the error, UpdateStatusWithAppIdEnvId", "err", err, "appId", installedApp.AppId, "envId", installedApp.EnvironmentId) - } - } - if isSucceeded { - // handling deployment success event - // updating cdWfr status - installedAppVersionHistory.Status = pipelineConfig.WorkflowSucceeded - installedAppVersionHistory.FinishedOn = time.Now() - installedAppVersionHistory.UpdatedOn = time.Now() - installedAppVersionHistory.UpdatedBy = 1 - installedAppVersionHistory, err = impl.installedAppVersionHistoryRepository.UpdateInstalledAppVersionHistory(installedAppVersionHistory, nil) - if err != nil { - impl.Logger.Errorw("error on update installedAppVersionHistory", "installedAppVersionHistory", installedAppVersionHistory, "err", err) - return err, isTimelineUpdated - } - - } - } - - return nil, isTimelineUpdated -} - -func (impl *CdHandlerImpl) CheckHelmAppStatusPeriodicallyAndUpdateInDb(helmPipelineStatusCheckEligibleTime int, getPipelineDeployedWithinHours int) error { - wfrList, err := impl.cdWorkflowRepository.GetLatestTriggersOfHelmPipelinesStuckInNonTerminalStatuses(getPipelineDeployedWithinHours) - if err != nil { - impl.Logger.Errorw("error in getting latest triggers of helm pipelines which are stuck in non terminal statuses", "err", err) - return err - } - impl.Logger.Debugw("checking helm app status for non terminal deployment triggers", "wfrList", wfrList, "number of wfr", len(wfrList)) - for _, wfr := range wfrList { - if time.Now().Sub(wfr.StartedOn) <= time.Duration(helmPipelineStatusCheckEligibleTime)*time.Second { - // if wfr is updated within configured time then do not include for this cron cycle - continue - } - appIdentifier := &client.AppIdentifier{ - ClusterId: wfr.CdWorkflow.Pipeline.Environment.ClusterId, - Namespace: wfr.CdWorkflow.Pipeline.Environment.Namespace, - ReleaseName: wfr.CdWorkflow.Pipeline.DeploymentAppName, - } - if isWfrUpdated := impl.workflowDagExecutor.UpdateWorkflowRunnerStatusForDeployment(appIdentifier, wfr, true); !isWfrUpdated { - continue - } - wfr.UpdatedBy = 1 - wfr.UpdatedOn = time.Now() - if wfr.Status == pipelineConfig.WorkflowFailed { - err = impl.pipelineStatusTimelineService.MarkPipelineStatusTimelineFailed(wfr.RefCdWorkflowRunnerId, pipelineConfig.NEW_DEPLOYMENT_INITIATED) - if err != nil { - impl.Logger.Errorw("error updating CdPipelineStatusTimeline", "err", err) - return err - } - } - err = impl.cdWorkflowRepository.UpdateWorkFlowRunner(wfr) - if err != nil { - impl.Logger.Errorw("error on update cd workflow runner", "wfr", wfr, "err", err) - return err - } - if slices.Contains(pipelineConfig.WfrTerminalStatusList, wfr.Status) { - util3.TriggerCDMetrics(pipelineConfig.GetTriggerMetricsFromRunnerObj(wfr), impl.config.ExposeCDMetrics) - } - - impl.Logger.Infow("updated workflow runner status for helm app", "wfr", wfr) - if wfr.Status == pipelineConfig.WorkflowSucceeded { - pipelineOverride, err := impl.pipelineOverrideRepository.FindLatestByCdWorkflowId(wfr.CdWorkflowId) - if err != nil { - impl.Logger.Errorw("error in getting latest pipeline override by cdWorkflowId", "err", err, "cdWorkflowId", wfr.CdWorkflowId) - return err - } - go impl.appService.WriteCDSuccessEvent(pipelineOverride.Pipeline.AppId, pipelineOverride.Pipeline.EnvironmentId, pipelineOverride) - err = impl.workflowDagExecutor.HandleDeploymentSuccessEvent(TriggerContext{}, pipelineOverride) - if err != nil { - impl.Logger.Errorw("error on handling deployment success event", "wfr", wfr, "err", err) - return err - } - } - } - return nil -} - func (impl *CdHandlerImpl) CancelStage(workflowRunnerId int, userId int32) (int, error) { workflowRunner, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(workflowRunnerId) if err != nil { @@ -1659,137 +1184,3 @@ func (impl *CdHandlerImpl) FetchAppDeploymentStatusForEnvironments(request resou func (impl *CdHandlerImpl) DeactivateImageReservationPathsOnFailure(imagePathReservationIds []int) error { return impl.customTagService.DeactivateImagePathReservationByImageIds(imagePathReservationIds) } - -func (impl *CdHandlerImpl) syncACDDevtronApps(deployedBeforeMinutes int, pipelineId int) error { - if impl.acdConfig.ArgoCDAutoSyncEnabled { - // don't check for apps if auto sync is enabled - return nil - } - cdWfr, err := impl.cdWorkflowRepository.FindLastStatusByPipelineIdAndRunnerType(pipelineId, bean.CD_WORKFLOW_TYPE_DEPLOY) - if err != nil { - impl.Logger.Errorw("error in getting latest cdWfr by cdPipelineId", "err", err, "pipelineId", pipelineId) - return err - } - if util3.IsTerminalStatus(cdWfr.Status) { - return nil - } - pipelineStatusTimeline, err := impl.pipelineStatusTimelineRepository.FetchLatestTimelineByWfrId(cdWfr.Id) - if err != nil { - impl.Logger.Errorw("error in fetching latest pipeline status by cdWfrId", "err", err) - return err - } - if pipelineStatusTimeline.Status == pipelineConfig.TIMELINE_STATUS_ARGOCD_SYNC_INITIATED && time.Since(pipelineStatusTimeline.StatusTime) >= time.Minute*time.Duration(deployedBeforeMinutes) { - acdToken, err := impl.argoUserService.GetLatestDevtronArgoCdUserToken() - if err != nil { - impl.Logger.Errorw("error in getting acd token", "err", err) - return err - } - ctx := context.Background() - ctx = context.WithValue(ctx, "token", acdToken) - syncTime := time.Now() - syncErr := impl.argocdClientWrapperService.SyncArgoCDApplicationIfNeededAndRefresh(ctx, cdWfr.CdWorkflow.Pipeline.DeploymentAppName) - if syncErr != nil { - impl.Logger.Errorw("error in syncing argoCD app", "err", syncErr) - timelineObject := impl.pipelineStatusTimelineService.GetTimelineDbObjectByTimelineStatusAndTimelineDescription(cdWfr.Id, 0, pipelineConfig.TIMELINE_STATUS_DEPLOYMENT_FAILED, fmt.Sprintf("error occured in syncing argocd application. err: %s", syncErr.Error()), 1, time.Now()) - _ = impl.pipelineStatusTimelineService.SaveTimeline(timelineObject, nil, false) - cdWfr.Status = pipelineConfig.WorkflowFailed - cdWfr.UpdatedBy = 1 - cdWfr.UpdatedOn = time.Now() - cdWfrUpdateErr := impl.cdWorkflowRepository.UpdateWorkFlowRunner(&cdWfr) - if cdWfrUpdateErr != nil { - impl.Logger.Errorw("error in updating cd workflow runner as failed in argocd app sync cron", "err", err) - return err - } - return nil - } - timeline := &pipelineConfig.PipelineStatusTimeline{ - CdWorkflowRunnerId: cdWfr.Id, - StatusTime: syncTime, - Status: pipelineConfig.TIMELINE_STATUS_ARGOCD_SYNC_COMPLETED, - StatusDetail: "argocd sync completed", - AuditLog: sql.AuditLog{ - CreatedBy: 1, - CreatedOn: time.Now(), - UpdatedBy: 1, - UpdatedOn: time.Now(), - }, - } - _, err, _ = impl.pipelineStatusTimelineService.SavePipelineStatusTimelineIfNotAlreadyPresent(timeline.CdWorkflowRunnerId, timeline.Status, timeline, false) - } - return nil -} - -func (impl *CdHandlerImpl) SyncACDHelmApps(deployedBeforeMinutes int, installedAppVersionId int) error { - if impl.acdConfig.ArgoCDAutoSyncEnabled { - // don't check for apps if auto sync is enabled - return nil - } - installedAppVersionHistory, err := impl.installedAppVersionHistoryRepository.GetLatestInstalledAppVersionHistory(installedAppVersionId) - if err != nil { - impl.Logger.Errorw("error in getting latest cdWfr by cdPipelineId", "err", err, "installedAppVersionId", installedAppVersionId) - return err - } - if util3.IsTerminalStatus(installedAppVersionHistory.Status) { - return nil - } - installedAppVersionHistoryId := installedAppVersionHistory.Id - pipelineStatusTimeline, err := impl.pipelineStatusTimelineRepository.FetchLatestTimelinesByInstalledAppVersionHistoryId(installedAppVersionHistoryId) - if err != nil { - impl.Logger.Errorw("error in fetching latest pipeline status by cdWfrId", "err", err) - return err - } - if pipelineStatusTimeline.Status == pipelineConfig.TIMELINE_STATUS_ARGOCD_SYNC_INITIATED && time.Since(pipelineStatusTimeline.StatusTime) >= time.Minute*time.Duration(deployedBeforeMinutes) { - installedApp, err := impl.installedAppRepository.GetInstalledAppByInstalledAppVersionId(installedAppVersionHistory.InstalledAppVersionId) - if err != nil { - impl.Logger.Errorw("error in fetching installed_app by installedAppVersionId", "err", err) - return err - } - appDetails, err := impl.appRepository.FindActiveById(installedApp.AppId) - if err != nil { - impl.Logger.Errorw("error in getting appDetails from appId", "err", err) - return err - } - envDetails, err := impl.envRepository.FindById(installedApp.EnvironmentId) - if err != nil { - impl.Logger.Errorw("error in fetching environment by envId", "err", err) - } - argoAppName := fmt.Sprintf("%s-%s", appDetails.AppName, envDetails.Name) - acdToken, err := impl.argoUserService.GetLatestDevtronArgoCdUserToken() - if err != nil { - impl.Logger.Errorw("error in getting acd token", "err", err) - return err - } - ctx := context.Background() - ctx = context.WithValue(ctx, "token", acdToken) - syncTime := time.Now() - syncErr := impl.argocdClientWrapperService.SyncArgoCDApplicationIfNeededAndRefresh(ctx, argoAppName) - if syncErr != nil { - impl.Logger.Errorw("error in syncing argoCD app", "err", syncErr) - timelineObject := impl.pipelineStatusTimelineService.GetTimelineDbObjectByTimelineStatusAndTimelineDescription(0, installedAppVersionHistoryId, pipelineConfig.TIMELINE_STATUS_DEPLOYMENT_FAILED, fmt.Sprintf("error occured in syncing argocd application. err: %s", syncErr.Error()), 1, time.Now()) - _ = impl.pipelineStatusTimelineService.SaveTimeline(timelineObject, nil, false) - installedAppVersionHistory.Status = pipelineConfig.WorkflowFailed - installedAppVersionHistory.UpdatedBy = 1 - installedAppVersionHistory.UpdatedOn = time.Now() - _, installedAppUpdateErr := impl.installedAppVersionHistoryRepository.UpdateInstalledAppVersionHistory(installedAppVersionHistory, nil) - if installedAppUpdateErr != nil { - impl.Logger.Errorw("error in updating cd workflow runner as failed in argocd app sync cron", "err", err) - return err - } - return nil - } - timeline := &pipelineConfig.PipelineStatusTimeline{ - InstalledAppVersionHistoryId: installedAppVersionHistoryId, - StatusTime: syncTime, - Status: pipelineConfig.TIMELINE_STATUS_ARGOCD_SYNC_COMPLETED, - StatusDetail: "argocd sync completed", - AuditLog: sql.AuditLog{ - CreatedBy: 1, - CreatedOn: time.Now(), - UpdatedBy: 1, - UpdatedOn: time.Now(), - }, - } - _, err, _ = impl.pipelineStatusTimelineService.SavePipelineStatusTimelineIfNotAlreadyPresent(timeline.CdWorkflowRunnerId, timeline.Status, timeline, false) - } - return nil -} diff --git a/pkg/pipeline/WebhookService.go b/pkg/pipeline/WebhookService.go index d09949749d..28ee023f48 100644 --- a/pkg/pipeline/WebhookService.go +++ b/pkg/pipeline/WebhookService.go @@ -18,33 +18,48 @@ package pipeline import ( - "bytes" "encoding/base64" "encoding/json" "fmt" "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/aws/aws-sdk-go-v2/service/ecr/types" pubsub "github.com/devtron-labs/common-lib/pubsub-lib" - "github.com/devtron-labs/devtron/client/events" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - util2 "github.com/devtron-labs/devtron/internal/util" - "github.com/devtron-labs/devtron/pkg/app" "github.com/devtron-labs/devtron/pkg/pipeline/bean" - "github.com/devtron-labs/devtron/pkg/pipeline/executors" - repository2 "github.com/devtron-labs/devtron/pkg/pipeline/repository" types2 "github.com/devtron-labs/devtron/pkg/pipeline/types" - repository3 "github.com/devtron-labs/devtron/pkg/plugin/repository" - "github.com/devtron-labs/devtron/pkg/sql" - "github.com/devtron-labs/devtron/util/event" - "github.com/go-pg/pg" + "github.com/devtron-labs/devtron/pkg/workflow/cd" + util3 "github.com/devtron-labs/devtron/util" "go.uber.org/zap" "strconv" "strings" - "sync" "time" ) +// this object's current object was previously used as CiCompleteEvent, duplicating it currently to remove unused fields here +type ExternalCiWebhookDto struct { + CiProjectDetails []bean.CiProjectDetails `json:"ciProjectDetails"` + DockerImage string `json:"dockerImage" validate:"required,image-validator"` + Digest string `json:"digest"` + PipelineId int `json:"pipelineId"` + WorkflowId *int `json:"workflowId"` + TriggeredBy int32 `json:"triggeredBy"` + PipelineName string `json:"pipelineName"` + DataSource string `json:"dataSource"` + MaterialType string `json:"materialType"` + Metrics util3.CIMetrics `json:"metrics"` + AppName string `json:"appName"` + IsArtifactUploaded bool `json:"isArtifactUploaded"` + FailureReason string `json:"failureReason"` + ImageDetailsFromCR *ImageDetailsFromCR `json:"imageDetailsFromCR"` + PluginRegistryArtifactDetails map[string][]string `json:"PluginRegistryArtifactDetails"` + PluginArtifactStage string `json:"pluginArtifactStage"` +} +type ImageDetailsFromCR struct { + ImageDetails []types.ImageDetail `json:"imageDetails"` + Region string `json:"region"` +} + type CiArtifactWebhookRequest struct { Image string `json:"image" validate:"required"` ImageDigest string `json:"imageDigest"` @@ -61,9 +76,6 @@ type CiArtifactWebhookRequest struct { type WebhookService interface { AuthenticateExternalCiWebhook(apiKey string) (int, error) - HandleCiSuccessEvent(triggerContext TriggerContext, ciPipelineId int, request *CiArtifactWebhookRequest, imagePushedAt *time.Time) (id int, err error) - HandleExternalCiWebhook(externalCiId int, request *CiArtifactWebhookRequest, auth func(token string, projectObject string, envObject string) bool, token string) (id int, err error) - HandleCiStepFailedEvent(ciPipelineId int, request *CiArtifactWebhookRequest) (err error) HandleMultipleImagesFromEvent(imageDetails []types.ImageDetail, ciWorkflowId int) (map[string]*pipelineConfig.CiWorkflow, error) GetTriggerValidateFuncs() []pubsub.ValidateMsg } @@ -74,40 +86,19 @@ type WebhookServiceImpl struct { logger *zap.SugaredLogger ciPipelineRepository pipelineConfig.CiPipelineRepository ciWorkflowRepository pipelineConfig.CiWorkflowRepository - appService app.AppService - eventClient client.EventClient - eventFactory client.EventFactory - workflowDagExecutor WorkflowDagExecutor - ciHandler CiHandler - pipelineStageRepository repository2.PipelineStageRepository - globalPluginRepository repository3.GlobalPluginRepository - customTagService CustomTagService + cdWorkflowCommonService cd.CdWorkflowCommonService } -func NewWebhookServiceImpl( - ciArtifactRepository repository.CiArtifactRepository, - logger *zap.SugaredLogger, - ciPipelineRepository pipelineConfig.CiPipelineRepository, - appService app.AppService, eventClient client.EventClient, - eventFactory client.EventFactory, +func NewWebhookServiceImpl(ciArtifactRepository repository.CiArtifactRepository, + logger *zap.SugaredLogger, ciPipelineRepository pipelineConfig.CiPipelineRepository, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, - workflowDagExecutor WorkflowDagExecutor, ciHandler CiHandler, - pipelineStageRepository repository2.PipelineStageRepository, - globalPluginRepository repository3.GlobalPluginRepository, - customTagService CustomTagService) *WebhookServiceImpl { + cdWorkflowCommonService cd.CdWorkflowCommonService) *WebhookServiceImpl { webhookHandler := &WebhookServiceImpl{ ciArtifactRepository: ciArtifactRepository, logger: logger, ciPipelineRepository: ciPipelineRepository, - appService: appService, - eventClient: eventClient, - eventFactory: eventFactory, ciWorkflowRepository: ciWorkflowRepository, - workflowDagExecutor: workflowDagExecutor, - ciHandler: ciHandler, - pipelineStageRepository: pipelineStageRepository, - globalPluginRepository: globalPluginRepository, - customTagService: customTagService, + cdWorkflowCommonService: cdWorkflowCommonService, } config, err := types2.GetCiConfig() if err != nil { @@ -145,326 +136,6 @@ func (impl WebhookServiceImpl) AuthenticateExternalCiWebhook(apiKey string) (int return id, nil } -func (impl WebhookServiceImpl) HandleCiStepFailedEvent(ciPipelineId int, request *CiArtifactWebhookRequest) (err error) { - - savedWorkflow, err := impl.ciWorkflowRepository.FindById(*request.WorkflowId) - if err != nil { - impl.logger.Errorw("cannot get saved wf", "wf ID: ", *request.WorkflowId, "err", err) - return err - } - - pipeline, err := impl.ciPipelineRepository.FindByCiAndAppDetailsById(ciPipelineId) - if err != nil { - impl.logger.Errorw("unable to find pipeline", "ID", ciPipelineId, "err", err) - return err - } - - go func() { - if len(savedWorkflow.ImagePathReservationIds) > 0 { - err = impl.customTagService.DeactivateImagePathReservationByImageIds(savedWorkflow.ImagePathReservationIds) - if err != nil { - impl.logger.Errorw("unable to deactivate impage_path_reservation ", err) - } - } - }() - - go impl.WriteCIStepFailedEvent(pipeline, request, savedWorkflow) - return nil -} - -func (impl WebhookServiceImpl) HandleCiSuccessEvent(triggerContext TriggerContext, ciPipelineId int, request *CiArtifactWebhookRequest, imagePushedAt *time.Time) (id int, err error) { - impl.logger.Infow("webhook for artifact save", "req", request) - if request.WorkflowId != nil { - savedWorkflow, err := impl.ciWorkflowRepository.FindById(*request.WorkflowId) - if err != nil { - impl.logger.Errorw("cannot get saved wf", "err", err) - return 0, err - } - // if workflow already cancelled then return, this state arises when user force aborts a ci - if savedWorkflow.Status == executors.WorkflowCancel { - return 0, err - } - savedWorkflow.Status = string(v1alpha1.NodeSucceeded) - impl.logger.Debugw("updating workflow ", "savedWorkflow", savedWorkflow) - err = impl.ciWorkflowRepository.UpdateWorkFlow(savedWorkflow) - if err != nil { - impl.logger.Errorw("update wf failed for id ", "err", err) - return 0, err - } - } - - pipeline, err := impl.ciPipelineRepository.FindByCiAndAppDetailsById(ciPipelineId) - if request.PipelineName == "" { - request.PipelineName = pipeline.Name - } - if err != nil { - impl.logger.Errorw("unable to find pipeline", "name", request.PipelineName, "err", err) - return 0, err - } - materialJson, err := request.MaterialInfo.MarshalJSON() - if err != nil { - impl.logger.Errorw("unable to marshal material metadata", "err", err) - return 0, err - } - dst := new(bytes.Buffer) - err = json.Compact(dst, materialJson) - if err != nil { - return 0, err - } - materialJson = dst.Bytes() - createdOn := time.Now() - updatedOn := time.Now() - if !imagePushedAt.IsZero() { - createdOn = *imagePushedAt - } - buildArtifact := &repository.CiArtifact{ - Image: request.Image, - ImageDigest: request.ImageDigest, - MaterialInfo: string(materialJson), - DataSource: request.DataSource, - PipelineId: pipeline.Id, - WorkflowId: request.WorkflowId, - ScanEnabled: pipeline.ScanEnabled, - Scanned: false, - IsArtifactUploaded: request.IsArtifactUploaded, - AuditLog: sql.AuditLog{CreatedBy: request.UserId, UpdatedBy: request.UserId, CreatedOn: createdOn, UpdatedOn: updatedOn}, - } - plugin, err := impl.globalPluginRepository.GetPluginByName(bean.VULNERABILITY_SCANNING_PLUGIN) - if err != nil || len(plugin) == 0 { - impl.logger.Errorw("error in getting image scanning plugin", "err", err) - return 0, err - } - isScanPluginConfigured, err := impl.pipelineStageRepository.CheckPluginExistsInCiPipeline(pipeline.Id, string(repository2.PIPELINE_STAGE_TYPE_POST_CI), plugin[0].Id) - if err != nil { - impl.logger.Errorw("error in getting ci pipeline plugin", "err", err, "pipelineId", pipeline.Id, "pluginId", plugin[0].Id) - return 0, err - } - if pipeline.ScanEnabled || isScanPluginConfigured { - buildArtifact.Scanned = true - buildArtifact.ScanEnabled = true - } - if err = impl.ciArtifactRepository.Save(buildArtifact); err != nil { - impl.logger.Errorw("error in saving material", "err", err) - return 0, err - } - - var pluginArtifacts []*repository.CiArtifact - for registry, artifacts := range request.PluginRegistryArtifactDetails { - for _, image := range artifacts { - if pipeline.PipelineType == bean.CI_JOB && image == "" { - continue - } - pluginArtifact := &repository.CiArtifact{ - Image: image, - ImageDigest: request.ImageDigest, - MaterialInfo: string(materialJson), - DataSource: request.PluginArtifactStage, - ComponentId: pipeline.Id, - PipelineId: pipeline.Id, - AuditLog: sql.AuditLog{CreatedBy: request.UserId, UpdatedBy: request.UserId, CreatedOn: createdOn, UpdatedOn: updatedOn}, - CredentialsSourceType: repository.GLOBAL_CONTAINER_REGISTRY, - CredentialSourceValue: registry, - ParentCiArtifact: buildArtifact.Id, - Scanned: buildArtifact.Scanned, - ScanEnabled: buildArtifact.ScanEnabled, - } - pluginArtifacts = append(pluginArtifacts, pluginArtifact) - } - } - if len(pluginArtifacts) > 0 { - _, err = impl.ciArtifactRepository.SaveAll(pluginArtifacts) - if err != nil { - impl.logger.Errorw("error while saving ci artifacts", "err", err) - return 0, err - } - } - - childrenCi, err := impl.ciPipelineRepository.FindByParentCiPipelineId(ciPipelineId) - if err != nil && !util2.IsErrNoRows(err) { - impl.logger.Errorw("error while fetching childern ci ", "err", err) - return 0, err - } - - var ciArtifactArr []*repository.CiArtifact - for _, ci := range childrenCi { - ciArtifact := &repository.CiArtifact{ - Image: request.Image, - ImageDigest: request.ImageDigest, - MaterialInfo: string(materialJson), - DataSource: request.DataSource, - PipelineId: ci.Id, - ParentCiArtifact: buildArtifact.Id, - ScanEnabled: ci.ScanEnabled, - Scanned: false, - IsArtifactUploaded: request.IsArtifactUploaded, - AuditLog: sql.AuditLog{CreatedBy: request.UserId, UpdatedBy: request.UserId, CreatedOn: time.Now(), UpdatedOn: time.Now()}, - } - if ci.ScanEnabled { - ciArtifact.Scanned = true - } - ciArtifactArr = append(ciArtifactArr, ciArtifact) - } - - impl.logger.Debugw("saving ci artifacts", "art", ciArtifactArr) - if len(ciArtifactArr) > 0 { - _, err = impl.ciArtifactRepository.SaveAll(ciArtifactArr) - if err != nil { - impl.logger.Errorw("error while saving ci artifacts", "err", err) - return 0, err - } - } - if len(pluginArtifacts) == 0 { - ciArtifactArr = append(ciArtifactArr, buildArtifact) - } else { - ciArtifactArr = append(ciArtifactArr, pluginArtifacts[0]) - } - go impl.WriteCISuccessEvent(request, pipeline, buildArtifact) - async := false - - // execute auto trigger in batch on CI success event - totalCIArtifactCount := len(ciArtifactArr) - batchSize := impl.ciConfig.CIAutoTriggerBatchSize - // handling to avoid infinite loop - if batchSize <= 0 { - batchSize = 1 - } - start := time.Now() - impl.logger.Infow("Started: auto trigger for children Stage/CD pipelines", "Artifact count", totalCIArtifactCount) - for i := 0; i < totalCIArtifactCount; { - // requests left to process - remainingBatch := totalCIArtifactCount - i - if remainingBatch < batchSize { - batchSize = remainingBatch - } - var wg sync.WaitGroup - for j := 0; j < batchSize; j++ { - wg.Add(1) - index := i + j - go func(index int) { - defer wg.Done() - ciArtifact := ciArtifactArr[index] - // handle individual CiArtifact success event - err = impl.workflowDagExecutor.HandleCiSuccessEvent(triggerContext, ciArtifact, async, request.UserId) - if err != nil { - impl.logger.Errorw("error on handle ci success event", "ciArtifactId", ciArtifact.Id, "err", err) - } - }(index) - } - wg.Wait() - i += batchSize - } - impl.logger.Debugw("Completed: auto trigger for children Stage/CD pipelines", "Time taken", time.Since(start).Seconds()) - return buildArtifact.Id, err -} - -func (impl WebhookServiceImpl) HandleExternalCiWebhook(externalCiId int, request *CiArtifactWebhookRequest, auth func(token string, projectObject string, envObject string) bool, token string) (id int, err error) { - externalCiPipeline, err := impl.ciPipelineRepository.FindExternalCiById(externalCiId) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in fetching external ci", "err", err) - return 0, err - } - if externalCiPipeline.Id == 0 { - impl.logger.Errorw("invalid external ci id", "externalCiId", externalCiId, "err", err) - return 0, &util2.ApiError{Code: "400", HttpStatusCode: 400, UserMessage: "invalid external ci id"} - } - - impl.logger.Infow("request of webhook external ci", "req", request) - materialJson, err := request.MaterialInfo.MarshalJSON() - if err != nil { - impl.logger.Errorw("unable to marshal material metadata", "err", err) - return 0, err - } - dst := new(bytes.Buffer) - err = json.Compact(dst, materialJson) - if err != nil { - impl.logger.Errorw("parsing error", "err", err) - return 0, err - } - materialJson = dst.Bytes() - artifact := &repository.CiArtifact{ - Image: request.Image, - ImageDigest: request.ImageDigest, - MaterialInfo: string(materialJson), - DataSource: request.DataSource, - WorkflowId: request.WorkflowId, - ExternalCiPipelineId: externalCiId, - ScanEnabled: false, - Scanned: false, - IsArtifactUploaded: request.IsArtifactUploaded, - AuditLog: sql.AuditLog{CreatedBy: request.UserId, UpdatedBy: request.UserId, CreatedOn: time.Now(), UpdatedOn: time.Now()}, - } - if err = impl.ciArtifactRepository.Save(artifact); err != nil { - impl.logger.Errorw("error in saving material", "err", err) - return 0, err - } - - hasAnyTriggered, err := impl.workflowDagExecutor.HandleWebhookExternalCiEvent(artifact, request.UserId, externalCiId, auth, token) - if err != nil { - impl.logger.Errorw("error on handle ext ci webhook", "err", err) - // if none of the child node has been triggered - if !hasAnyTriggered { - if err1 := impl.ciArtifactRepository.Delete(artifact); err1 != nil { - impl.logger.Errorw("error in rollback artifact", "err", err1) - return 0, err1 - } - } - } - return artifact.Id, err -} - -func (impl *WebhookServiceImpl) WriteCIStepFailedEvent(pipeline *pipelineConfig.CiPipeline, request *CiArtifactWebhookRequest, ciWorkflow *pipelineConfig.CiWorkflow) { - event := impl.eventFactory.Build(util.Fail, &pipeline.Id, pipeline.AppId, nil, util.CI) - material := &client.MaterialTriggerInfo{} - material.GitTriggers = ciWorkflow.GitTriggers - event.CiWorkflowRunnerId = ciWorkflow.Id - event.UserId = int(ciWorkflow.TriggeredBy) - event = impl.eventFactory.BuildExtraCIData(event, material, request.Image) - event.CiArtifactId = 0 - event.Payload.FailureReason = request.FailureReason - _, evtErr := impl.eventClient.WriteNotificationEvent(event) - if evtErr != nil { - impl.logger.Errorw("error in writing event: ", event, "error: ", evtErr) - } -} - -func (impl *WebhookServiceImpl) WriteCISuccessEvent(request *CiArtifactWebhookRequest, pipeline *pipelineConfig.CiPipeline, artifact *repository.CiArtifact) { - event := impl.eventFactory.Build(util.Success, &pipeline.Id, pipeline.AppId, nil, util.CI) - event.CiArtifactId = artifact.Id - if artifact.WorkflowId != nil { - event.CiWorkflowRunnerId = *artifact.WorkflowId - } - event.UserId = int(request.UserId) - event = impl.eventFactory.BuildExtraCIData(event, nil, artifact.Image) - _, evtErr := impl.eventClient.WriteNotificationEvent(event) - if evtErr != nil { - impl.logger.Errorw("error in writing event", "err", evtErr) - } -} - -func (impl *WebhookServiceImpl) BuildPayload(request *CiArtifactWebhookRequest, pipeline *pipelineConfig.CiPipeline) *client.Payload { - payload := &client.Payload{} - payload.AppName = pipeline.App.AppName - payload.PipelineName = pipeline.Name - - var ciMaterials []*repository.CiMaterialInfo - err := json.Unmarshal(request.MaterialInfo, &ciMaterials) - if err != nil { - impl.logger.Errorw("err", "err", err) - } - - for _, material := range ciMaterials { - if material.Modifications != nil && len(material.Modifications) > 0 { - revision := material.Modifications[0].Revision - if payload.Source == "" { - payload.Source = revision - } - payload.Source = payload.Source + "," + revision - } - } - payload.DockerImageUrl = request.Image - return payload -} - // HandleMultipleImagesFromEvent handles multiple images from plugin and creates ci workflow for n-1 images for mapping in ci_artifact func (impl *WebhookServiceImpl) HandleMultipleImagesFromEvent(imageDetails []types.ImageDetail, ciWorkflowId int) (map[string]*pipelineConfig.CiWorkflow, error) { ciWorkflow, err := impl.ciWorkflowRepository.FindById(ciWorkflowId) @@ -507,5 +178,5 @@ func (impl *WebhookServiceImpl) HandleMultipleImagesFromEvent(imageDetails []typ } func (impl *WebhookServiceImpl) GetTriggerValidateFuncs() []pubsub.ValidateMsg { - return impl.workflowDagExecutor.GetTriggerValidateFuncs() + return impl.cdWorkflowCommonService.GetTriggerValidateFuncs() } diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go deleted file mode 100644 index 57e18f5a60..0000000000 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ /dev/null @@ -1,2579 +0,0 @@ -/* - * Copyright (c) 2020 Devtron Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package pipeline - -import ( - "context" - "encoding/json" - "fmt" - bean6 "github.com/devtron-labs/devtron/api/helm-app/bean" - client2 "github.com/devtron-labs/devtron/api/helm-app/service" - "github.com/devtron-labs/devtron/pkg/build/artifacts" - "github.com/devtron-labs/devtron/pkg/deployment/manifest" - "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate" - "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps" - bean7 "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" - "github.com/devtron-labs/devtron/pkg/eventProcessor/out" - "github.com/devtron-labs/devtron/pkg/workflow/cd" - "strconv" - "strings" - "sync" - "time" - - blob_storage "github.com/devtron-labs/common-lib/blob-storage" - "github.com/devtron-labs/common-lib/pubsub-lib/model" - "github.com/devtron-labs/common-lib/utils/k8s/health" - "github.com/devtron-labs/devtron/client/argocdServer/application" - gitSensorClient "github.com/devtron-labs/devtron/client/gitSensor" - "github.com/devtron-labs/devtron/pkg/app/status" - "github.com/devtron-labs/devtron/pkg/auth/user" - "github.com/devtron-labs/devtron/pkg/imageDigestPolicy" - bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" - repository4 "github.com/devtron-labs/devtron/pkg/pipeline/repository" - "github.com/devtron-labs/devtron/pkg/pipeline/types" - "github.com/devtron-labs/devtron/pkg/plugin" - "github.com/devtron-labs/devtron/pkg/resourceQualifiers" - serverBean "github.com/devtron-labs/devtron/pkg/server/bean" - "github.com/devtron-labs/devtron/pkg/variables" - repository5 "github.com/devtron-labs/devtron/pkg/variables/repository" - util4 "github.com/devtron-labs/devtron/util" - "github.com/devtron-labs/devtron/util/argo" - "github.com/pkg/errors" - "go.opentelemetry.io/otel" - "k8s.io/utils/strings/slices" - - "github.com/devtron-labs/devtron/internal/sql/repository/appWorkflow" - repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" - history2 "github.com/devtron-labs/devtron/pkg/pipeline/history" - repository3 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" - "github.com/devtron-labs/devtron/pkg/sql" - util3 "github.com/devtron-labs/devtron/pkg/util" - - pubsub "github.com/devtron-labs/common-lib/pubsub-lib" - "github.com/devtron-labs/devtron/api/bean" - client "github.com/devtron-labs/devtron/client/events" - "github.com/devtron-labs/devtron/internal/sql/models" - "github.com/devtron-labs/devtron/internal/sql/repository" - "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/devtron-labs/devtron/internal/sql/repository/security" - "github.com/devtron-labs/devtron/internal/util" - "github.com/devtron-labs/devtron/pkg/app" - bean2 "github.com/devtron-labs/devtron/pkg/bean" - util2 "github.com/devtron-labs/devtron/util/event" - "github.com/devtron-labs/devtron/util/rbac" - "github.com/go-pg/pg" - "go.uber.org/zap" -) - -type WorkflowDagExecutor interface { - HandleCiSuccessEvent(triggerContext TriggerContext, artifact *repository.CiArtifact, async bool, triggeredBy int32) error - HandleWebhookExternalCiEvent(artifact *repository.CiArtifact, triggeredBy int32, externalCiId int, auth func(token string, projectObject string, envObject string) bool, token string) (bool, error) - HandlePreStageSuccessEvent(triggerContext TriggerContext, cdStageCompleteEvent bean7.CdStageCompleteEvent) error - HandleDeploymentSuccessEvent(triggerContext TriggerContext, pipelineOverride *chartConfig.PipelineOverride) error - HandlePostStageSuccessEvent(triggerContext TriggerContext, cdWorkflowId int, cdPipelineId int, triggeredBy int32, pluginRegistryImageDetails map[string][]string) error - - TriggerPostStage(request TriggerRequest) error - TriggerPreStage(request TriggerRequest) error - TriggerDeployment(request TriggerRequest) error - ManualCdTrigger(triggerContext TriggerContext, overrideRequest *bean.ValuesOverrideRequest) (int, error) - TriggerStageForBulk(triggerRequest TriggerRequest) error - - TriggerBulkHibernateAsync(request StopDeploymentGroupRequest, ctx context.Context) (interface{}, error) - StopStartApp(triggerContext TriggerContext, stopRequest *StopAppRequest) (int, error) - - MarkCurrentDeploymentFailed(runner *pipelineConfig.CdWorkflowRunner, releaseErr error, triggeredBy int32) error - UpdateWorkflowRunnerStatusForDeployment(appIdentifier *client2.AppIdentifier, wfr *pipelineConfig.CdWorkflowRunner, skipReleaseNotFound bool) bool - OnDeleteCdPipelineEvent(pipelineId int, triggeredBy int32) - GetTriggerValidateFuncs() []pubsub.ValidateMsg -} - -type WorkflowDagExecutorImpl struct { - logger *zap.SugaredLogger - pipelineRepository pipelineConfig.PipelineRepository - cdWorkflowRepository pipelineConfig.CdWorkflowRepository - pubsubClient *pubsub.PubSubClientServiceImpl - cdWorkflowService WorkflowService - ciPipelineRepository pipelineConfig.CiPipelineRepository - materialRepository pipelineConfig.MaterialRepository - pipelineOverrideRepository chartConfig.PipelineOverrideRepository - ciArtifactRepository repository.CiArtifactRepository - user user.UserService - enforcerUtil rbac.EnforcerUtil - groupRepository repository.DeploymentGroupRepository - envRepository repository2.EnvironmentRepository - eventFactory client.EventFactory - eventClient client.EventClient - cvePolicyRepository security.CvePolicyRepository - scanResultRepository security.ImageScanResultRepository - appWorkflowRepository appWorkflow.AppWorkflowRepository - prePostCdScriptHistoryService history2.PrePostCdScriptHistoryService - argoUserService argo.ArgoUserService - cdPipelineStatusTimelineRepo pipelineConfig.PipelineStatusTimelineRepository - pipelineStatusTimelineService status.PipelineStatusTimelineService - CiTemplateRepository pipelineConfig.CiTemplateRepository - ciWorkflowRepository pipelineConfig.CiWorkflowRepository - appLabelRepository pipelineConfig.AppLabelRepository - gitSensorGrpcClient gitSensorClient.Client - pipelineStageService PipelineStageService - config *types.CdConfig - appServiceConfig *app.AppServiceConfig - globalPluginService plugin.GlobalPluginService - - scopedVariableManager variables.ScopedVariableCMCSManager - pluginInputVariableParser PluginInputVariableParser - - devtronAsyncHelmInstallRequestMap map[int]bool - devtronAsyncHelmInstallRequestLock *sync.Mutex - devtronAppReleaseContextMap map[int]DevtronAppReleaseContextType - devtronAppReleaseContextMapLock *sync.Mutex - - helmAppService client2.HelmAppService - customTagService CustomTagService - imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService - - cdWorkflowCommonService cd.CdWorkflowCommonService - cdTriggerService devtronApps.TriggerService - - deployedConfigurationHistoryService history2.DeployedConfigurationHistoryService - workflowEventPublishService out.WorkflowEventPublishService - manifestCreationService manifest.ManifestCreationService - deploymentTemplateService deploymentTemplate.DeploymentTemplateService - commonArtifactService artifacts.CommonArtifactService -} - -const ( - GIT_COMMIT_HASH_PREFIX = "GIT_COMMIT_HASH" - GIT_SOURCE_TYPE_PREFIX = "GIT_SOURCE_TYPE" - GIT_SOURCE_VALUE_PREFIX = "GIT_SOURCE_VALUE" - GIT_SOURCE_COUNT = "GIT_SOURCE_COUNT" - APP_LABEL_KEY_PREFIX = "APP_LABEL_KEY" - APP_LABEL_VALUE_PREFIX = "APP_LABEL_VALUE" - APP_LABEL_COUNT = "APP_LABEL_COUNT" - CHILD_CD_ENV_NAME_PREFIX = "CHILD_CD_ENV_NAME" - CHILD_CD_CLUSTER_NAME_PREFIX = "CHILD_CD_CLUSTER_NAME" - CHILD_CD_COUNT = "CHILD_CD_COUNT" - DEVTRON_SYSTEM_USER_ID = 1 - ARGOCD_REFRESH_ERROR = "Error in refreshing argocd app" -) - -type DevtronAppReleaseContextType struct { - CancelContext context.CancelFunc - RunnerId int -} - -type TriggerRequest struct { - CdWf *pipelineConfig.CdWorkflow - Pipeline *pipelineConfig.Pipeline - Artifact *repository.CiArtifact - ApplyAuth bool - TriggeredBy int32 - RefCdWorkflowRunnerId int - TriggerContext -} - -type TriggerContext struct { - // Context is a context object to be passed to the pipeline trigger - // +optional - Context context.Context - // ReferenceId is a unique identifier for the workflow runner - // refer pipelineConfig.CdWorkflowRunner - ReferenceId *string -} - -func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pipelineConfig.PipelineRepository, - cdWorkflowRepository pipelineConfig.CdWorkflowRepository, - pubsubClient *pubsub.PubSubClientServiceImpl, - cdWorkflowService WorkflowService, - ciArtifactRepository repository.CiArtifactRepository, - ciPipelineRepository pipelineConfig.CiPipelineRepository, - materialRepository pipelineConfig.MaterialRepository, - pipelineOverrideRepository chartConfig.PipelineOverrideRepository, - user user.UserService, - groupRepository repository.DeploymentGroupRepository, - envRepository repository2.EnvironmentRepository, - enforcerUtil rbac.EnforcerUtil, eventFactory client.EventFactory, - eventClient client.EventClient, cvePolicyRepository security.CvePolicyRepository, - scanResultRepository security.ImageScanResultRepository, - appWorkflowRepository appWorkflow.AppWorkflowRepository, - prePostCdScriptHistoryService history2.PrePostCdScriptHistoryService, - argoUserService argo.ArgoUserService, - cdPipelineStatusTimelineRepo pipelineConfig.PipelineStatusTimelineRepository, - pipelineStatusTimelineService status.PipelineStatusTimelineService, - CiTemplateRepository pipelineConfig.CiTemplateRepository, - ciWorkflowRepository pipelineConfig.CiWorkflowRepository, - appLabelRepository pipelineConfig.AppLabelRepository, gitSensorGrpcClient gitSensorClient.Client, - pipelineStageService PipelineStageService, - globalPluginService plugin.GlobalPluginService, - pluginInputVariableParser PluginInputVariableParser, - scopedVariableManager variables.ScopedVariableCMCSManager, - helmAppService client2.HelmAppService, - pipelineConfigListenerService PipelineConfigListenerService, - customTagService CustomTagService, - imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService, - cdWorkflowCommonService cd.CdWorkflowCommonService, - cdTriggerService devtronApps.TriggerService, - deployedConfigurationHistoryService history2.DeployedConfigurationHistoryService, - workflowEventPublishService out.WorkflowEventPublishService, - manifestCreationService manifest.ManifestCreationService, - deploymentTemplateService deploymentTemplate.DeploymentTemplateService, - commonArtifactService artifacts.CommonArtifactService) *WorkflowDagExecutorImpl { - wde := &WorkflowDagExecutorImpl{logger: Logger, - pipelineRepository: pipelineRepository, - cdWorkflowRepository: cdWorkflowRepository, - pubsubClient: pubsubClient, - cdWorkflowService: cdWorkflowService, - ciPipelineRepository: ciPipelineRepository, - ciArtifactRepository: ciArtifactRepository, - materialRepository: materialRepository, - pipelineOverrideRepository: pipelineOverrideRepository, - user: user, - enforcerUtil: enforcerUtil, - groupRepository: groupRepository, - envRepository: envRepository, - eventFactory: eventFactory, - eventClient: eventClient, - cvePolicyRepository: cvePolicyRepository, - scanResultRepository: scanResultRepository, - appWorkflowRepository: appWorkflowRepository, - prePostCdScriptHistoryService: prePostCdScriptHistoryService, - argoUserService: argoUserService, - cdPipelineStatusTimelineRepo: cdPipelineStatusTimelineRepo, - pipelineStatusTimelineService: pipelineStatusTimelineService, - CiTemplateRepository: CiTemplateRepository, - ciWorkflowRepository: ciWorkflowRepository, - appLabelRepository: appLabelRepository, - gitSensorGrpcClient: gitSensorGrpcClient, - pipelineStageService: pipelineStageService, - scopedVariableManager: scopedVariableManager, - globalPluginService: globalPluginService, - pluginInputVariableParser: pluginInputVariableParser, - - devtronAsyncHelmInstallRequestMap: make(map[int]bool), - devtronAsyncHelmInstallRequestLock: &sync.Mutex{}, - devtronAppReleaseContextMap: make(map[int]DevtronAppReleaseContextType), - devtronAppReleaseContextMapLock: &sync.Mutex{}, - helmAppService: helmAppService, - customTagService: customTagService, - imageDigestPolicyService: imageDigestPolicyService, - cdWorkflowCommonService: cdWorkflowCommonService, - cdTriggerService: cdTriggerService, - deployedConfigurationHistoryService: deployedConfigurationHistoryService, - workflowEventPublishService: workflowEventPublishService, - manifestCreationService: manifestCreationService, - deploymentTemplateService: deploymentTemplateService, - commonArtifactService: commonArtifactService, - } - config, err := types.GetCdConfig() - if err != nil { - return nil - } - wde.config = config - appServiceConfig, err := app.GetAppServiceConfig() - if err != nil { - return nil - } - wde.appServiceConfig = appServiceConfig - err = wde.SubscribeDevtronAsyncHelmInstallRequest() - if err != nil { - return nil - } - pipelineConfigListenerService.RegisterPipelineDeleteListener(wde) - return wde -} - -func (impl *WorkflowDagExecutorImpl) extractOverrideRequestFromCDAsyncInstallEvent(msg *model.PubSubMsg) (*bean7.AsyncCdDeployEvent, *client2.AppIdentifier, error) { - CDAsyncInstallNatsMessage := &bean7.AsyncCdDeployEvent{} - err := json.Unmarshal([]byte(msg.Data), CDAsyncInstallNatsMessage) - if err != nil { - impl.logger.Errorw("error in unmarshalling CD async install request nats message", "err", err) - return nil, nil, err - } - pipeline, err := impl.pipelineRepository.FindById(CDAsyncInstallNatsMessage.ValuesOverrideRequest.PipelineId) - if err != nil { - impl.logger.Errorw("error in fetching pipeline by pipelineId", "err", err) - return nil, nil, err - } - impl.SetPipelineFieldsInOverrideRequest(CDAsyncInstallNatsMessage.ValuesOverrideRequest, pipeline) - if CDAsyncInstallNatsMessage.ValuesOverrideRequest.DeploymentType == models.DEPLOYMENTTYPE_UNKNOWN { - CDAsyncInstallNatsMessage.ValuesOverrideRequest.DeploymentType = models.DEPLOYMENTTYPE_DEPLOY - } - appIdentifier := &client2.AppIdentifier{ - ClusterId: pipeline.Environment.ClusterId, - Namespace: pipeline.Environment.Namespace, - ReleaseName: pipeline.DeploymentAppName, - } - return CDAsyncInstallNatsMessage, appIdentifier, nil -} - -// UpdateWorkflowRunnerStatusForDeployment will update CD workflow runner based on release status and app status -func (impl *WorkflowDagExecutorImpl) UpdateWorkflowRunnerStatusForDeployment(appIdentifier *client2.AppIdentifier, wfr *pipelineConfig.CdWorkflowRunner, skipReleaseNotFound bool) bool { - helmInstalledDevtronApp, err := impl.helmAppService.GetApplicationAndReleaseStatus(context.Background(), appIdentifier) - if err != nil { - impl.logger.Errorw("error in getting helm app release status", "appIdentifier", appIdentifier, "err", err) - // Handle release not found errors - if skipReleaseNotFound && util.GetGRPCErrorDetailedMessage(err) != bean6.ErrReleaseNotFound { - // skip this error and continue for next workflow status - impl.logger.Warnw("found error, skipping helm apps status update for this trigger", "appIdentifier", appIdentifier, "err", err) - return false - } - // If release not found, mark the deployment as failure - wfr.Status = pipelineConfig.WorkflowFailed - wfr.Message = util.GetGRPCErrorDetailedMessage(err) - wfr.FinishedOn = time.Now() - return true - } - - switch helmInstalledDevtronApp.GetReleaseStatus() { - case serverBean.HelmReleaseStatusSuperseded: - // If release status is superseded, mark the deployment as failure - wfr.Status = pipelineConfig.WorkflowFailed - wfr.Message = pipelineConfig.NEW_DEPLOYMENT_INITIATED - wfr.FinishedOn = time.Now() - return true - case serverBean.HelmReleaseStatusFailed: - // If release status is failed, mark the deployment as failure - wfr.Status = pipelineConfig.WorkflowFailed - wfr.Message = helmInstalledDevtronApp.GetDescription() - wfr.FinishedOn = time.Now() - return true - case serverBean.HelmReleaseStatusDeployed: - //skip if there is no deployment after wfr.StartedOn and continue for next workflow status - if helmInstalledDevtronApp.GetLastDeployed().AsTime().Before(wfr.StartedOn) { - impl.logger.Warnw("release mismatched, skipping helm apps status update for this trigger", "appIdentifier", appIdentifier, "err", err) - return false - } - - if helmInstalledDevtronApp.GetApplicationStatus() == application.Healthy { - // mark the deployment as succeed - wfr.Status = pipelineConfig.WorkflowSucceeded - wfr.FinishedOn = time.Now() - return true - } - } - if wfr.Status == pipelineConfig.WorkflowInProgress { - return false - } - wfr.Status = pipelineConfig.WorkflowInProgress - return true -} - -func (impl *WorkflowDagExecutorImpl) handleAsyncTriggerReleaseError(releaseErr error, cdWfr *pipelineConfig.CdWorkflowRunner, overrideRequest *bean.ValuesOverrideRequest, appIdentifier *client2.AppIdentifier) { - releaseErrString := util.GetGRPCErrorDetailedMessage(releaseErr) - switch releaseErrString { - case context.DeadlineExceeded.Error(): - // if context deadline is exceeded fetch release status and UpdateWorkflowRunnerStatusForDeployment - if isWfrUpdated := impl.UpdateWorkflowRunnerStatusForDeployment(appIdentifier, cdWfr, false); !isWfrUpdated { - // updating cdWfr to failed - if err := impl.MarkCurrentDeploymentFailed(cdWfr, fmt.Errorf("Deployment timeout: release %s took more than %d mins", appIdentifier.ReleaseName, impl.appServiceConfig.DevtronChartInstallRequestTimeout), overrideRequest.UserId); err != nil { - impl.logger.Errorw("error while updating current runner status to failed, handleAsyncTriggerReleaseError", "cdWfr", cdWfr.Id, "err", err) - } - } - cdWfr.UpdatedBy = 1 - cdWfr.UpdatedOn = time.Now() - err := impl.cdWorkflowRepository.UpdateWorkFlowRunner(cdWfr) - if err != nil { - impl.logger.Errorw("error on update cd workflow runner", "wfr", cdWfr, "err", err) - return - } - cdMetrics := util4.CDMetrics{ - AppName: cdWfr.CdWorkflow.Pipeline.DeploymentAppName, - Status: cdWfr.Status, - DeploymentType: cdWfr.CdWorkflow.Pipeline.DeploymentAppType, - EnvironmentName: cdWfr.CdWorkflow.Pipeline.Environment.Name, - Time: time.Since(cdWfr.StartedOn).Seconds() - time.Since(cdWfr.FinishedOn).Seconds(), - } - util4.TriggerCDMetrics(cdMetrics, impl.config.ExposeCDMetrics) - impl.logger.Infow("updated workflow runner status for helm app", "wfr", cdWfr) - return - case context.Canceled.Error(): - if err := impl.MarkCurrentDeploymentFailed(cdWfr, errors.New(pipelineConfig.NEW_DEPLOYMENT_INITIATED), overrideRequest.UserId); err != nil { - impl.logger.Errorw("error while updating current runner status to failed, handleAsyncTriggerReleaseError", "cdWfr", cdWfr.Id, "err", err) - } - return - case "": - return - default: - if err := impl.MarkCurrentDeploymentFailed(cdWfr, releaseErr, overrideRequest.UserId); err != nil { - impl.logger.Errorw("error while updating current runner status to failed, handleAsyncTriggerReleaseError", "cdWfr", cdWfr.Id, "err", err) - } - return - } -} - -func (impl *WorkflowDagExecutorImpl) handleIfPreviousRunnerTriggerRequest(currentRunner *pipelineConfig.CdWorkflowRunner, userId int32) (bool, error) { - exists, err := impl.cdWorkflowRepository.IsLatestCDWfr(currentRunner.Id, currentRunner.CdWorkflow.PipelineId) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("err on fetching latest cd workflow runner, SubscribeDevtronAsyncHelmInstallRequest", "err", err) - return false, err - } - return exists, nil -} - -func (impl *WorkflowDagExecutorImpl) UpdateReleaseContextForPipeline(pipelineId, cdWfrId int, cancel context.CancelFunc) { - impl.devtronAppReleaseContextMapLock.Lock() - defer impl.devtronAppReleaseContextMapLock.Unlock() - if releaseContext, ok := impl.devtronAppReleaseContextMap[pipelineId]; ok { - //Abort previous running release - impl.logger.Infow("new deployment has been triggered with a running deployment in progress!", "aborting deployment for pipelineId", pipelineId) - releaseContext.CancelContext() - } - impl.devtronAppReleaseContextMap[pipelineId] = DevtronAppReleaseContextType{ - CancelContext: cancel, - RunnerId: cdWfrId, - } -} - -func (impl *WorkflowDagExecutorImpl) RemoveReleaseContextForPipeline(pipelineId int, triggeredBy int32) { - impl.devtronAppReleaseContextMapLock.Lock() - defer impl.devtronAppReleaseContextMapLock.Unlock() - if releaseContext, ok := impl.devtronAppReleaseContextMap[pipelineId]; ok { - //Abort previous running release - impl.logger.Infow("CD pipeline has been deleted with a running deployment in progress!", "aborting deployment for pipelineId", pipelineId) - cdWfr, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(releaseContext.RunnerId) - if err != nil { - impl.logger.Errorw("err on fetching cd workflow runner, RemoveReleaseContextForPipeline", "err", err) - } - if err = impl.MarkCurrentDeploymentFailed(cdWfr, errors.New("CD pipeline has been deleted"), triggeredBy); err != nil { - impl.logger.Errorw("error while updating current runner status to failed, RemoveReleaseContextForPipeline", "cdWfr", cdWfr.Id, "err", err) - } - releaseContext.CancelContext() - delete(impl.devtronAppReleaseContextMap, pipelineId) - } - return -} - -func (impl *WorkflowDagExecutorImpl) OnDeleteCdPipelineEvent(pipelineId int, triggeredBy int32) { - impl.logger.Debugw("CD pipeline delete event received", "pipelineId", pipelineId, "deletedBy", triggeredBy) - impl.RemoveReleaseContextForPipeline(pipelineId, triggeredBy) - return -} - -func (impl *WorkflowDagExecutorImpl) isReleaseContextExistsForPipeline(pipelineId, cdWfrId int) bool { - impl.devtronAppReleaseContextMapLock.Lock() - defer impl.devtronAppReleaseContextMapLock.Unlock() - if releaseContext, ok := impl.devtronAppReleaseContextMap[pipelineId]; ok { - return releaseContext.RunnerId == cdWfrId - } - return false -} - -func (impl *WorkflowDagExecutorImpl) handleConcurrentRequest(wfrId int) bool { - impl.devtronAsyncHelmInstallRequestLock.Lock() - defer impl.devtronAsyncHelmInstallRequestLock.Unlock() - if _, exists := impl.devtronAsyncHelmInstallRequestMap[wfrId]; exists { - //request is in process already, Skip here - return true - } - impl.devtronAsyncHelmInstallRequestMap[wfrId] = true - return false -} - -func (impl *WorkflowDagExecutorImpl) cleanUpDevtronAppReleaseContextMap(pipelineId, wfrId int) { - if impl.isReleaseContextExistsForPipeline(pipelineId, wfrId) { - impl.devtronAppReleaseContextMapLock.Lock() - defer impl.devtronAppReleaseContextMapLock.Unlock() - if _, ok := impl.devtronAppReleaseContextMap[pipelineId]; ok { - delete(impl.devtronAppReleaseContextMap, pipelineId) - } - } -} - -func (impl *WorkflowDagExecutorImpl) cleanUpDevtronAsyncHelmInstallRequest(pipelineId, wfrId int) { - impl.devtronAsyncHelmInstallRequestLock.Lock() - defer impl.devtronAsyncHelmInstallRequestLock.Unlock() - if _, exists := impl.devtronAsyncHelmInstallRequestMap[wfrId]; exists { - //request is in process already, Skip here - delete(impl.devtronAsyncHelmInstallRequestMap, wfrId) - } - impl.cleanUpDevtronAppReleaseContextMap(pipelineId, wfrId) -} - -func (impl *WorkflowDagExecutorImpl) processDevtronAsyncHelmInstallRequest(CDAsyncInstallNatsMessage *bean7.AsyncCdDeployEvent, appIdentifier *client2.AppIdentifier) { - overrideRequest := CDAsyncInstallNatsMessage.ValuesOverrideRequest - cdWfr, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(overrideRequest.WfrId) - if err != nil { - impl.logger.Errorw("err on fetching cd workflow runner, processDevtronAsyncHelmInstallRequest", "err", err) - return - } - - // skip if the cdWfr.Status is already in a terminal state - skipCDWfrStatusList := pipelineConfig.WfrTerminalStatusList - skipCDWfrStatusList = append(skipCDWfrStatusList, pipelineConfig.WorkflowInProgress) - if slices.Contains(skipCDWfrStatusList, cdWfr.Status) { - impl.logger.Warnw("skipped deployment as the workflow runner status is already in terminal state, processDevtronAsyncHelmInstallRequest", "cdWfrId", cdWfr.Id, "status", cdWfr.Status) - return - } - - //skip if the cdWfr is not the latest one - exists, err := impl.handleIfPreviousRunnerTriggerRequest(cdWfr, overrideRequest.UserId) - if err != nil { - impl.logger.Errorw("err in validating latest cd workflow runner, processDevtronAsyncHelmInstallRequest", "err", err) - return - } - if exists { - impl.logger.Warnw("skipped deployment as the workflow runner is not the latest one", "cdWfrId", cdWfr.Id) - err := impl.MarkCurrentDeploymentFailed(cdWfr, errors.New(pipelineConfig.NEW_DEPLOYMENT_INITIATED), overrideRequest.UserId) - if err != nil { - impl.logger.Errorw("error while updating current runner status to failed, processDevtronAsyncHelmInstallRequest", "cdWfr", cdWfr.Id, "err", err) - return - } - return - } - - if cdWfr.Status == pipelineConfig.WorkflowStarting && impl.isReleaseContextExistsForPipeline(overrideRequest.PipelineId, cdWfr.Id) { - impl.logger.Warnw("event redelivered! deployment is currently in progress, processDevtronAsyncHelmInstallRequest", "cdWfrId", cdWfr.Id, "status", cdWfr.Status) - return - } - - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(impl.appServiceConfig.DevtronChartInstallRequestTimeout)*time.Minute) - defer cancel() - - impl.UpdateReleaseContextForPipeline(overrideRequest.PipelineId, cdWfr.Id, cancel) - //update workflow runner status, used in app workflow view - err = impl.cdWorkflowCommonService.UpdateCDWorkflowRunnerStatus(ctx, overrideRequest, CDAsyncInstallNatsMessage.TriggeredAt, pipelineConfig.WorkflowStarting, "") - if err != nil { - impl.logger.Errorw("error in updating the workflow runner status, processDevtronAsyncHelmInstallRequest", "cdWfrId", cdWfr.Id, "err", err) - return - } - // build merged values and save PCO history for the release - valuesOverrideResponse, builtChartPath, err := impl.manifestCreationService.BuildManifestForTrigger(overrideRequest, CDAsyncInstallNatsMessage.TriggeredAt, ctx) - if err != nil { - return - } - - _, span := otel.Tracer("orchestrator").Start(ctx, "appService.TriggerRelease") - releaseId, _, releaseErr := impl.cdTriggerService.TriggerRelease(overrideRequest, valuesOverrideResponse, builtChartPath, ctx, CDAsyncInstallNatsMessage.TriggeredAt, CDAsyncInstallNatsMessage.TriggeredBy) - span.End() - if releaseErr != nil { - impl.handleAsyncTriggerReleaseError(releaseErr, cdWfr, overrideRequest, appIdentifier) - } else { - impl.logger.Infow("pipeline triggered successfully !!", "cdPipelineId", overrideRequest.PipelineId, "artifactId", overrideRequest.CiArtifactId, "releaseId", releaseId) - // Update previous deployment runner status (in transaction): Failed - _, span = otel.Tracer("orchestrator").Start(ctx, "updatePreviousDeploymentStatus") - err1 := impl.updatePreviousDeploymentStatus(cdWfr, overrideRequest.PipelineId, CDAsyncInstallNatsMessage.TriggeredAt, overrideRequest.UserId) - span.End() - if err1 != nil { - impl.logger.Errorw("error while update previous cd workflow runners, processDevtronAsyncHelmInstallRequest", "err", err, "runner", cdWfr, "pipelineId", overrideRequest.PipelineId) - return - } - } -} - -func (impl *WorkflowDagExecutorImpl) SubscribeDevtronAsyncHelmInstallRequest() error { - callback := func(msg *model.PubSubMsg) { - CDAsyncInstallNatsMessage, appIdentifier, err := impl.extractOverrideRequestFromCDAsyncInstallEvent(msg) - if err != nil { - impl.logger.Errorw("err on extracting override request, SubscribeDevtronAsyncHelmInstallRequest", "err", err) - return - } - if skip := impl.handleConcurrentRequest(CDAsyncInstallNatsMessage.ValuesOverrideRequest.WfrId); skip { - impl.logger.Warnw("concurrent request received, SubscribeDevtronAsyncHelmInstallRequest", "WfrId", CDAsyncInstallNatsMessage.ValuesOverrideRequest.WfrId) - return - } - defer impl.cleanUpDevtronAsyncHelmInstallRequest(CDAsyncInstallNatsMessage.ValuesOverrideRequest.PipelineId, CDAsyncInstallNatsMessage.ValuesOverrideRequest.WfrId) - impl.processDevtronAsyncHelmInstallRequest(CDAsyncInstallNatsMessage, appIdentifier) - return - } - - // add required logging here - var loggerFunc pubsub.LoggerFunc = func(msg model.PubSubMsg) (string, []interface{}) { - CDAsyncInstallNatsMessage := &bean7.AsyncCdDeployEvent{} - err := json.Unmarshal([]byte(msg.Data), CDAsyncInstallNatsMessage) - if err != nil { - return "error in unmarshalling CD async install request nats message", []interface{}{"err", err} - } - return "got message for devtron chart install", []interface{}{"appId", CDAsyncInstallNatsMessage.ValuesOverrideRequest.AppId, "pipelineId", CDAsyncInstallNatsMessage.ValuesOverrideRequest.PipelineId, "artifactId", CDAsyncInstallNatsMessage.ValuesOverrideRequest.CiArtifactId} - } - - err := impl.pubsubClient.Subscribe(pubsub.DEVTRON_CHART_INSTALL_TOPIC, callback, loggerFunc) - if err != nil { - impl.logger.Error(err) - return err - } - return nil -} - -func (impl *WorkflowDagExecutorImpl) HandleCiSuccessEvent(triggerContext TriggerContext, artifact *repository.CiArtifact, async bool, triggeredBy int32) error { - //1. get cd pipelines - //2. get config - //3. trigger wf/ deployment - var pipelineID int - if artifact.DataSource == repository.POST_CI { - pipelineID = artifact.ComponentId - } else { - // TODO: need to migrate artifact.PipelineId for dataSource="CI_RUNNER" also to component_id - pipelineID = artifact.PipelineId - } - pipelines, err := impl.pipelineRepository.FindByParentCiPipelineId(pipelineID) - if err != nil { - impl.logger.Errorw("error in fetching cd pipeline", "pipelineId", artifact.PipelineId, "err", err) - return err - } - for _, pipeline := range pipelines { - triggerRequest := TriggerRequest{ - CdWf: nil, - Pipeline: pipeline, - Artifact: artifact, - TriggeredBy: triggeredBy, - TriggerContext: triggerContext, - } - err = impl.triggerIfAutoStageCdPipeline(triggerRequest) - if err != nil { - impl.logger.Debugw("error on trigger cd pipeline", "err", err) - } - } - return nil -} - -func (impl *WorkflowDagExecutorImpl) HandleWebhookExternalCiEvent(artifact *repository.CiArtifact, triggeredBy int32, externalCiId int, auth func(token string, projectObject string, envObject string) bool, token string) (bool, error) { - hasAnyTriggered := false - appWorkflowMappings, err := impl.appWorkflowRepository.FindWFCDMappingByExternalCiId(externalCiId) - if err != nil { - impl.logger.Errorw("error in fetching cd pipeline", "pipelineId", artifact.PipelineId, "err", err) - return hasAnyTriggered, err - } - - var pipelines []*pipelineConfig.Pipeline - for _, appWorkflowMapping := range appWorkflowMappings { - pipeline, err := impl.pipelineRepository.FindById(appWorkflowMapping.ComponentId) - if err != nil { - impl.logger.Errorw("error in fetching cd pipeline", "pipelineId", artifact.PipelineId, "err", err) - return hasAnyTriggered, err - } - projectObject := impl.enforcerUtil.GetAppRBACNameByAppId(pipeline.AppId) - envObject := impl.enforcerUtil.GetAppRBACByAppIdAndPipelineId(pipeline.AppId, pipeline.Id) - if !auth(token, projectObject, envObject) { - err = &util.ApiError{Code: "401", HttpStatusCode: 401, UserMessage: "Unauthorized"} - return hasAnyTriggered, err - } - if pipeline.TriggerType == pipelineConfig.TRIGGER_TYPE_MANUAL { - impl.logger.Warnw("skipping deployment for manual trigger for webhook", "pipeline", pipeline) - continue - } - pipelines = append(pipelines, pipeline) - } - - for _, pipeline := range pipelines { - //applyAuth=false, already auth applied for this flow - triggerRequest := TriggerRequest{ - CdWf: nil, - Pipeline: pipeline, - Artifact: artifact, - ApplyAuth: false, - TriggeredBy: triggeredBy, - } - err = impl.triggerIfAutoStageCdPipeline(triggerRequest) - if err != nil { - impl.logger.Debugw("error on trigger cd pipeline", "err", err) - return hasAnyTriggered, err - } - hasAnyTriggered = true - } - - return hasAnyTriggered, err -} - -// if stage is present with 0 stage steps, delete the stage -// handle corrupt data (https://github.com/devtron-labs/devtron/issues/3826) -func (impl *WorkflowDagExecutorImpl) deleteCorruptedPipelineStage(pipelineStage *repository4.PipelineStage, triggeredBy int32) (error, bool) { - if pipelineStage != nil { - stageReq := &bean3.PipelineStageDto{ - Id: pipelineStage.Id, - Type: pipelineStage.Type, - } - err, deleted := impl.pipelineStageService.DeletePipelineStageIfReq(stageReq, triggeredBy) - if err != nil { - impl.logger.Errorw("error in deleting the corrupted pipeline stage", "err", err, "pipelineStageReq", stageReq) - return err, false - } - return nil, deleted - } - return nil, false -} - -func (impl *WorkflowDagExecutorImpl) triggerIfAutoStageCdPipeline(request TriggerRequest) error { - - preStage, err := impl.getPipelineStage(request.Pipeline.Id, repository4.PIPELINE_STAGE_TYPE_PRE_CD) - if err != nil { - return err - } - - //handle corrupt data (https://github.com/devtron-labs/devtron/issues/3826) - err, deleted := impl.deleteCorruptedPipelineStage(preStage, request.TriggeredBy) - if err != nil { - impl.logger.Errorw("error in deleteCorruptedPipelineStage ", "cdPipelineId", request.Pipeline.Id, "err", err, "preStage", preStage, "triggeredBy", request.TriggeredBy) - return err - } - - request.TriggerContext.Context = context.Background() - if len(request.Pipeline.PreStageConfig) > 0 || (preStage != nil && !deleted) { - // pre stage exists - if request.Pipeline.PreTriggerType == pipelineConfig.TRIGGER_TYPE_AUTOMATIC { - impl.logger.Debugw("trigger pre stage for pipeline", "artifactId", request.Artifact.Id, "pipelineId", request.Pipeline.Id) - err = impl.TriggerPreStage(request) // TODO handle error here - return err - } - } else if request.Pipeline.TriggerType == pipelineConfig.TRIGGER_TYPE_AUTOMATIC { - // trigger deployment - impl.logger.Debugw("trigger cd for pipeline", "artifactId", request.Artifact.Id, "pipelineId", request.Pipeline.Id) - err = impl.TriggerDeployment(request) - return err - } - return nil -} - -func (impl *WorkflowDagExecutorImpl) getPipelineStage(pipelineId int, stageType repository4.PipelineStageType) (*repository4.PipelineStage, error) { - stage, err := impl.pipelineStageService.GetCdStageByCdPipelineIdAndStageType(pipelineId, stageType) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in fetching CD pipeline stage", "cdPipelineId", pipelineId, "stage ", stage, "err", err) - return nil, err - } - return stage, nil -} - -func (impl *WorkflowDagExecutorImpl) TriggerStageForBulk(triggerRequest TriggerRequest) error { - - preStage, err := impl.getPipelineStage(triggerRequest.Pipeline.Id, repository4.PIPELINE_STAGE_TYPE_PRE_CD) - if err != nil { - return err - } - - //handle corrupt data (https://github.com/devtron-labs/devtron/issues/3826) - err, deleted := impl.deleteCorruptedPipelineStage(preStage, triggerRequest.TriggeredBy) - if err != nil { - impl.logger.Errorw("error in deleteCorruptedPipelineStage ", "cdPipelineId", triggerRequest.Pipeline.Id, "err", err, "preStage", preStage, "triggeredBy", triggerRequest.TriggeredBy) - return err - } - - triggerRequest.TriggerContext.Context = context.Background() - if len(triggerRequest.Pipeline.PreStageConfig) > 0 || (preStage != nil && !deleted) { - //pre stage exists - impl.logger.Debugw("trigger pre stage for pipeline", "artifactId", triggerRequest.Artifact.Id, "pipelineId", triggerRequest.Pipeline.Id) - triggerRequest.RefCdWorkflowRunnerId = 0 - err = impl.TriggerPreStage(triggerRequest) // TODO handle error here - return err - } else { - // trigger deployment - impl.logger.Debugw("trigger cd for pipeline", "artifactId", triggerRequest.Artifact.Id, "pipelineId", triggerRequest.Pipeline.Id) - err = impl.TriggerDeployment(triggerRequest) - return err - } -} - -func (impl *WorkflowDagExecutorImpl) HandlePreStageSuccessEvent(triggerContext TriggerContext, cdStageCompleteEvent bean7.CdStageCompleteEvent) error { - wfRunner, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(cdStageCompleteEvent.WorkflowRunnerId) - if err != nil { - return err - } - if wfRunner.WorkflowType == bean.CD_WORKFLOW_TYPE_PRE { - pipeline, err := impl.pipelineRepository.FindById(cdStageCompleteEvent.CdPipelineId) - if err != nil { - return err - } - ciArtifact, err := impl.ciArtifactRepository.Get(cdStageCompleteEvent.CiArtifactDTO.Id) - if err != nil { - return err - } - // Migration of deprecated DataSource Type - if ciArtifact.IsMigrationRequired() { - migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(ciArtifact.Id) - if migrationErr != nil { - impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", ciArtifact.Id) - } - } - PreCDArtifacts, err := impl.commonArtifactService.SavePluginArtifacts(ciArtifact, cdStageCompleteEvent.PluginRegistryArtifactDetails, pipeline.Id, repository.PRE_CD, cdStageCompleteEvent.TriggeredBy) - if err != nil { - impl.logger.Errorw("error in saving plugin artifacts", "err", err) - return err - } - if pipeline.TriggerType == pipelineConfig.TRIGGER_TYPE_AUTOMATIC { - if len(PreCDArtifacts) > 0 { - ciArtifact = PreCDArtifacts[0] // deployment will be trigger with artifact copied by plugin - } - cdWorkflow, err := impl.cdWorkflowRepository.FindById(cdStageCompleteEvent.WorkflowId) - if err != nil { - return err - } - //passing applyAuth as false since this event is for auto trigger and user who already has access to this cd can trigger pre cd also - applyAuth := false - if cdStageCompleteEvent.TriggeredBy != 1 { - applyAuth = true - } - triggerRequest := TriggerRequest{ - CdWf: cdWorkflow, - Pipeline: pipeline, - Artifact: ciArtifact, - ApplyAuth: applyAuth, - TriggeredBy: cdStageCompleteEvent.TriggeredBy, - TriggerContext: triggerContext, - } - triggerRequest.TriggerContext.Context = context.Background() - err = impl.TriggerDeployment(triggerRequest) - if err != nil { - return err - } - } - } - return nil -} - -func (impl *WorkflowDagExecutorImpl) TriggerPreStage(request TriggerRequest) error { - //setting triggeredAt variable to have consistent data for various audit log places in db for deployment time - triggeredAt := time.Now() - triggeredBy := request.TriggeredBy - artifact := request.Artifact - pipeline := request.Pipeline - ctx := request.TriggerContext.Context - //in case of pre stage manual trigger auth is already applied and for auto triggers there is no need for auth check here - cdWf := request.CdWf - var err error - if cdWf == nil { - cdWf = &pipelineConfig.CdWorkflow{ - CiArtifactId: artifact.Id, - PipelineId: pipeline.Id, - AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: 1, UpdatedOn: triggeredAt, UpdatedBy: 1}, - } - err = impl.cdWorkflowRepository.SaveWorkFlow(ctx, cdWf) - if err != nil { - return err - } - } - cdWorkflowExecutorType := impl.config.GetWorkflowExecutorType() - runner := &pipelineConfig.CdWorkflowRunner{ - Name: pipeline.Name, - WorkflowType: bean.CD_WORKFLOW_TYPE_PRE, - ExecutorType: cdWorkflowExecutorType, - Status: pipelineConfig.WorkflowStarting, // starting PreStage - TriggeredBy: triggeredBy, - StartedOn: triggeredAt, - Namespace: impl.config.GetDefaultNamespace(), - BlobStorageEnabled: impl.config.BlobStorageEnabled, - CdWorkflowId: cdWf.Id, - LogLocation: fmt.Sprintf("%s/%s%s-%s/main.log", impl.config.GetDefaultBuildLogsKeyPrefix(), strconv.Itoa(cdWf.Id), string(bean.CD_WORKFLOW_TYPE_PRE), pipeline.Name), - AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: 1, UpdatedOn: triggeredAt, UpdatedBy: 1}, - RefCdWorkflowRunnerId: request.RefCdWorkflowRunnerId, - ReferenceId: request.TriggerContext.ReferenceId, - } - var env *repository2.Environment - if pipeline.RunPreStageInEnv { - _, span := otel.Tracer("orchestrator").Start(ctx, "envRepository.FindById") - env, err = impl.envRepository.FindById(pipeline.EnvironmentId) - span.End() - if err != nil { - impl.logger.Errorw(" unable to find env ", "err", err) - return err - } - impl.logger.Debugw("env", "env", env) - runner.Namespace = env.Namespace - } - _, span := otel.Tracer("orchestrator").Start(ctx, "cdWorkflowRepository.SaveWorkFlowRunner") - _, err = impl.cdWorkflowRepository.SaveWorkFlowRunner(runner) - span.End() - if err != nil { - return err - } - - //checking vulnerability for the selected image - isVulnerable, err := impl.GetArtifactVulnerabilityStatus(artifact, pipeline, ctx) - if err != nil { - impl.logger.Errorw("error in getting Artifact vulnerability status, TriggerPreStage", "err", err) - return err - } - if isVulnerable { - // if image vulnerable, update timeline status and return - runner.Status = pipelineConfig.WorkflowFailed - runner.Message = pipelineConfig.FOUND_VULNERABILITY - runner.FinishedOn = time.Now() - runner.UpdatedOn = time.Now() - runner.UpdatedBy = triggeredBy - err = impl.cdWorkflowRepository.UpdateWorkFlowRunner(runner) - if err != nil { - impl.logger.Errorw("error in updating wfr status due to vulnerable image", "err", err) - return err - } - return fmt.Errorf("found vulnerability for image digest %s", artifact.ImageDigest) - } - - _, span = otel.Tracer("orchestrator").Start(ctx, "buildWFRequest") - cdStageWorkflowRequest, err := impl.buildWFRequest(runner, cdWf, pipeline, triggeredBy) - span.End() - if err != nil { - return err - } - cdStageWorkflowRequest.StageType = types.PRE - // handling copyContainerImage plugin specific logic - imagePathReservationIds, err := impl.SetCopyContainerImagePluginDataInWorkflowRequest(cdStageWorkflowRequest, pipeline.Id, types.PRE, artifact) - if err != nil { - runner.Status = pipelineConfig.WorkflowFailed - runner.Message = err.Error() - _ = impl.cdWorkflowRepository.UpdateWorkFlowRunner(runner) - return err - } else { - runner.ImagePathReservationIds = imagePathReservationIds - _ = impl.cdWorkflowRepository.UpdateWorkFlowRunner(runner) - } - - _, span = otel.Tracer("orchestrator").Start(ctx, "cdWorkflowService.SubmitWorkflow") - cdStageWorkflowRequest.Pipeline = pipeline - cdStageWorkflowRequest.Env = env - cdStageWorkflowRequest.Type = bean3.CD_WORKFLOW_PIPELINE_TYPE - _, err = impl.cdWorkflowService.SubmitWorkflow(cdStageWorkflowRequest) - span.End() - err = impl.sendPreStageNotification(ctx, cdWf, pipeline) - if err != nil { - return err - } - //creating cd config history entry - _, span = otel.Tracer("orchestrator").Start(ctx, "prePostCdScriptHistoryService.CreatePrePostCdScriptHistory") - err = impl.prePostCdScriptHistoryService.CreatePrePostCdScriptHistory(pipeline, nil, repository3.PRE_CD_TYPE, true, triggeredBy, triggeredAt) - span.End() - if err != nil { - impl.logger.Errorw("error in creating pre cd script entry", "err", err, "pipeline", pipeline) - return err - } - return nil -} - -func (impl *WorkflowDagExecutorImpl) SetCopyContainerImagePluginDataInWorkflowRequest(cdStageWorkflowRequest *types.WorkflowRequest, pipelineId int, pipelineStage string, artifact *repository.CiArtifact) ([]int, error) { - copyContainerImagePluginId, err := impl.globalPluginService.GetRefPluginIdByRefPluginName(COPY_CONTAINER_IMAGE) - var imagePathReservationIds []int - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in getting copyContainerImage plugin id", "err", err) - return imagePathReservationIds, err - } - for _, step := range cdStageWorkflowRequest.PrePostDeploySteps { - if copyContainerImagePluginId != 0 && step.RefPluginId == copyContainerImagePluginId { - var pipelineStageEntityType int - if pipelineStage == types.PRE { - pipelineStageEntityType = bean3.EntityTypePreCD - } else { - pipelineStageEntityType = bean3.EntityTypePostCD - } - customTagId := -1 - var DockerImageTag string - - customTag, err := impl.customTagService.GetActiveCustomTagByEntityKeyAndValue(pipelineStageEntityType, strconv.Itoa(pipelineId)) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in fetching custom tag data", "err", err) - return imagePathReservationIds, err - } - - if !customTag.Enabled { - // case when custom tag is not configured - source image tag will be taken as docker image tag - pluginTriggerImageSplit := strings.Split(artifact.Image, ":") - DockerImageTag = pluginTriggerImageSplit[len(pluginTriggerImageSplit)-1] - } else { - // for copyContainerImage plugin parse destination images and save its data in image path reservation table - customTagDbObject, customDockerImageTag, err := impl.customTagService.GetCustomTag(pipelineStageEntityType, strconv.Itoa(pipelineId)) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in fetching custom tag by entity key and value for CD", "err", err) - return imagePathReservationIds, err - } - if customTagDbObject != nil && customTagDbObject.Id > 0 { - customTagId = customTagDbObject.Id - } - DockerImageTag = customDockerImageTag - } - - var sourceDockerRegistryId string - if artifact.DataSource == repository.PRE_CD || artifact.DataSource == repository.POST_CD || artifact.DataSource == repository.POST_CI { - if artifact.CredentialsSourceType == repository.GLOBAL_CONTAINER_REGISTRY { - sourceDockerRegistryId = artifact.CredentialSourceValue - } - } else { - sourceDockerRegistryId = cdStageWorkflowRequest.DockerRegistryId - } - registryDestinationImageMap, registryCredentialMap, err := impl.pluginInputVariableParser.HandleCopyContainerImagePluginInputVariables(step.InputVars, DockerImageTag, cdStageWorkflowRequest.CiArtifactDTO.Image, sourceDockerRegistryId) - if err != nil { - impl.logger.Errorw("error in parsing copyContainerImage input variable", "err", err) - return imagePathReservationIds, err - } - var destinationImages []string - for _, images := range registryDestinationImageMap { - for _, image := range images { - destinationImages = append(destinationImages, image) - } - } - // fetch already saved artifacts to check if they are already present - savedCIArtifacts, err := impl.ciArtifactRepository.FindCiArtifactByImagePaths(destinationImages) - if err != nil { - impl.logger.Errorw("error in fetching artifacts by image path", "err", err) - return imagePathReservationIds, err - } - if len(savedCIArtifacts) > 0 { - // if already present in ci artifact, return "image path already in use error" - return imagePathReservationIds, bean3.ErrImagePathInUse - } - imagePathReservationIds, err = impl.ReserveImagesGeneratedAtPlugin(customTagId, registryDestinationImageMap) - if err != nil { - impl.logger.Errorw("error in reserving image", "err", err) - return imagePathReservationIds, err - } - cdStageWorkflowRequest.RegistryDestinationImageMap = registryDestinationImageMap - cdStageWorkflowRequest.RegistryCredentialMap = registryCredentialMap - var pluginArtifactStage string - if pipelineStage == types.PRE { - pluginArtifactStage = repository.PRE_CD - } else { - pluginArtifactStage = repository.POST_CD - } - cdStageWorkflowRequest.PluginArtifactStage = pluginArtifactStage - } - } - return imagePathReservationIds, nil -} - -func (impl *WorkflowDagExecutorImpl) sendPreStageNotification(ctx context.Context, cdWf *pipelineConfig.CdWorkflow, pipeline *pipelineConfig.Pipeline) error { - wfr, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(ctx, cdWf.Id, bean.CD_WORKFLOW_TYPE_PRE) - if err != nil { - return err - } - - event := impl.eventFactory.Build(util2.Trigger, &pipeline.Id, pipeline.AppId, &pipeline.EnvironmentId, util2.CD) - impl.logger.Debugw("event PreStageTrigger", "event", event) - event = impl.eventFactory.BuildExtraCDData(event, &wfr, 0, bean.CD_WORKFLOW_TYPE_PRE) - _, span := otel.Tracer("orchestrator").Start(ctx, "eventClient.WriteNotificationEvent") - _, evtErr := impl.eventClient.WriteNotificationEvent(event) - span.End() - if evtErr != nil { - impl.logger.Errorw("CD trigger event not sent", "error", evtErr) - } - return nil -} - -func convert(ts string) (*time.Time, error) { - //layout := "2006-01-02T15:04:05Z" - t, err := time.Parse(bean2.LayoutRFC3339, ts) - if err != nil { - return nil, err - } - return &t, nil -} - -func (impl *WorkflowDagExecutorImpl) TriggerPostStage(request TriggerRequest) error { - //setting triggeredAt variable to have consistent data for various audit log places in db for deployment time - triggeredAt := time.Now() - triggeredBy := request.TriggeredBy - pipeline := request.Pipeline - cdWf := request.CdWf - - runner := &pipelineConfig.CdWorkflowRunner{ - Name: pipeline.Name, - WorkflowType: bean.CD_WORKFLOW_TYPE_POST, - ExecutorType: impl.config.GetWorkflowExecutorType(), - Status: pipelineConfig.WorkflowStarting, // starting PostStage - TriggeredBy: triggeredBy, - StartedOn: triggeredAt, - Namespace: impl.config.GetDefaultNamespace(), - BlobStorageEnabled: impl.config.BlobStorageEnabled, - CdWorkflowId: cdWf.Id, - LogLocation: fmt.Sprintf("%s/%s%s-%s/main.log", impl.config.GetDefaultBuildLogsKeyPrefix(), strconv.Itoa(cdWf.Id), string(bean.CD_WORKFLOW_TYPE_POST), pipeline.Name), - AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: triggeredBy, UpdatedOn: triggeredAt, UpdatedBy: triggeredBy}, - RefCdWorkflowRunnerId: request.RefCdWorkflowRunnerId, - ReferenceId: request.TriggerContext.ReferenceId, - } - var env *repository2.Environment - var err error - if pipeline.RunPostStageInEnv { - env, err = impl.envRepository.FindById(pipeline.EnvironmentId) - if err != nil { - impl.logger.Errorw(" unable to find env ", "err", err) - return err - } - runner.Namespace = env.Namespace - } - - _, err = impl.cdWorkflowRepository.SaveWorkFlowRunner(runner) - if err != nil { - return err - } - - if cdWf.CiArtifact == nil || cdWf.CiArtifact.Id == 0 { - cdWf.CiArtifact, err = impl.ciArtifactRepository.Get(cdWf.CiArtifactId) - if err != nil { - impl.logger.Errorw("error fetching artifact data", "err", err) - return err - } - } - // Migration of deprecated DataSource Type - if cdWf.CiArtifact.IsMigrationRequired() { - migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(cdWf.CiArtifact.Id) - if migrationErr != nil { - impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", cdWf.CiArtifact.Id) - } - } - //checking vulnerability for the selected image - isVulnerable, err := impl.GetArtifactVulnerabilityStatus(cdWf.CiArtifact, pipeline, context.Background()) - if err != nil { - impl.logger.Errorw("error in getting Artifact vulnerability status, TriggerPostStage", "err", err) - return err - } - if isVulnerable { - // if image vulnerable, update timeline status and return - runner.Status = pipelineConfig.WorkflowFailed - runner.Message = pipelineConfig.FOUND_VULNERABILITY - runner.FinishedOn = time.Now() - runner.UpdatedOn = time.Now() - runner.UpdatedBy = triggeredBy - err = impl.cdWorkflowRepository.UpdateWorkFlowRunner(runner) - if err != nil { - impl.logger.Errorw("error in updating wfr status due to vulnerable image", "err", err) - return err - } - return fmt.Errorf("found vulnerability for image digest %s", cdWf.CiArtifact.ImageDigest) - } - - cdStageWorkflowRequest, err := impl.buildWFRequest(runner, cdWf, pipeline, triggeredBy) - if err != nil { - impl.logger.Errorw("error in building wfRequest", "err", err, "runner", runner, "cdWf", cdWf, "pipeline", pipeline) - return err - } - cdStageWorkflowRequest.StageType = types.POST - cdStageWorkflowRequest.Pipeline = pipeline - cdStageWorkflowRequest.Env = env - cdStageWorkflowRequest.Type = bean3.CD_WORKFLOW_PIPELINE_TYPE - // handling plugin specific logic - - pluginImagePathReservationIds, err := impl.SetCopyContainerImagePluginDataInWorkflowRequest(cdStageWorkflowRequest, pipeline.Id, types.POST, cdWf.CiArtifact) - if err != nil { - runner.Status = pipelineConfig.WorkflowFailed - runner.Message = err.Error() - _ = impl.cdWorkflowRepository.UpdateWorkFlowRunner(runner) - return err - } - - _, err = impl.cdWorkflowService.SubmitWorkflow(cdStageWorkflowRequest) - if err != nil { - impl.logger.Errorw("error in submitting workflow", "err", err, "cdStageWorkflowRequest", cdStageWorkflowRequest, "pipeline", pipeline, "env", env) - return err - } - - wfr, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(context.Background(), cdWf.Id, bean.CD_WORKFLOW_TYPE_POST) - if err != nil { - impl.logger.Errorw("error in getting wfr by workflowId and runnerType", "err", err, "wfId", cdWf.Id) - return err - } - wfr.ImagePathReservationIds = pluginImagePathReservationIds - err = impl.cdWorkflowRepository.UpdateWorkFlowRunner(&wfr) - if err != nil { - impl.logger.Error("error in updating image path reservation ids in cd workflow runner", "err", "err") - } - - event := impl.eventFactory.Build(util2.Trigger, &pipeline.Id, pipeline.AppId, &pipeline.EnvironmentId, util2.CD) - impl.logger.Debugw("event Cd Post Trigger", "event", event) - event = impl.eventFactory.BuildExtraCDData(event, &wfr, 0, bean.CD_WORKFLOW_TYPE_POST) - _, evtErr := impl.eventClient.WriteNotificationEvent(event) - if evtErr != nil { - impl.logger.Errorw("CD trigger event not sent", "error", evtErr) - } - //creating cd config history entry - err = impl.prePostCdScriptHistoryService.CreatePrePostCdScriptHistory(pipeline, nil, repository3.POST_CD_TYPE, true, triggeredBy, triggeredAt) - if err != nil { - impl.logger.Errorw("error in creating post cd script entry", "err", err, "pipeline", pipeline) - return err - } - return nil -} - -func (impl *WorkflowDagExecutorImpl) ReserveImagesGeneratedAtPlugin(customTagId int, registryImageMap map[string][]string) ([]int, error) { - var imagePathReservationIds []int - for _, images := range registryImageMap { - for _, image := range images { - imagePathReservationData, err := impl.customTagService.ReserveImagePath(image, customTagId) - if err != nil { - impl.logger.Errorw("Error in marking custom tag reserved", "err", err) - return imagePathReservationIds, err - } - if imagePathReservationData != nil { - imagePathReservationIds = append(imagePathReservationIds, imagePathReservationData.Id) - } - } - } - return imagePathReservationIds, nil -} - -func (impl *WorkflowDagExecutorImpl) buildArtifactLocationForS3(cdWorkflowConfig *pipelineConfig.CdWorkflowConfig, cdWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner) (string, string, string) { - cdArtifactLocationFormat := cdWorkflowConfig.CdArtifactLocationFormat - if cdArtifactLocationFormat == "" { - cdArtifactLocationFormat = impl.config.GetArtifactLocationFormat() - } - if cdWorkflowConfig.LogsBucket == "" { - cdWorkflowConfig.LogsBucket = impl.config.GetDefaultBuildLogsBucket() - } - ArtifactLocation := fmt.Sprintf("s3://%s/"+impl.config.GetDefaultArtifactKeyPrefix()+"/"+cdArtifactLocationFormat, cdWorkflowConfig.LogsBucket, cdWf.Id, runner.Id) - artifactFileName := fmt.Sprintf(impl.config.GetDefaultArtifactKeyPrefix()+"/"+cdArtifactLocationFormat, cdWf.Id, runner.Id) - return ArtifactLocation, cdWorkflowConfig.LogsBucket, artifactFileName -} - -func (impl *WorkflowDagExecutorImpl) getDeployStageDetails(pipelineId int) (pipelineConfig.CdWorkflowRunner, string, int, error) { - deployStageWfr := pipelineConfig.CdWorkflowRunner{} - //getting deployment pipeline latest wfr by pipelineId - deployStageWfr, err := impl.cdWorkflowRepository.FindLastStatusByPipelineIdAndRunnerType(pipelineId, bean.CD_WORKFLOW_TYPE_DEPLOY) - if err != nil { - impl.logger.Errorw("error in getting latest status of deploy type wfr by pipelineId", "err", err, "pipelineId", pipelineId) - return deployStageWfr, "", 0, err - } - deployStageTriggeredByUserEmail, err := impl.user.GetEmailById(deployStageWfr.TriggeredBy) - if err != nil { - impl.logger.Errorw("error in getting user email by id", "err", err, "userId", deployStageWfr.TriggeredBy) - return deployStageWfr, "", 0, err - } - pipelineReleaseCounter, err := impl.pipelineOverrideRepository.GetCurrentPipelineReleaseCounter(pipelineId) - if err != nil { - impl.logger.Errorw("error occurred while fetching latest release counter for pipeline", "pipelineId", pipelineId, "err", err) - return deployStageWfr, "", 0, err - } - return deployStageWfr, deployStageTriggeredByUserEmail, pipelineReleaseCounter, nil -} - -func isExtraVariableDynamic(variableName string, webhookAndCiData *gitSensorClient.WebhookAndCiData) bool { - if strings.Contains(variableName, GIT_COMMIT_HASH_PREFIX) || strings.Contains(variableName, GIT_SOURCE_TYPE_PREFIX) || strings.Contains(variableName, GIT_SOURCE_VALUE_PREFIX) || - strings.Contains(variableName, APP_LABEL_VALUE_PREFIX) || strings.Contains(variableName, APP_LABEL_KEY_PREFIX) || - strings.Contains(variableName, CHILD_CD_ENV_NAME_PREFIX) || strings.Contains(variableName, CHILD_CD_CLUSTER_NAME_PREFIX) || - strings.Contains(variableName, CHILD_CD_COUNT) || strings.Contains(variableName, APP_LABEL_COUNT) || strings.Contains(variableName, GIT_SOURCE_COUNT) || - webhookAndCiData != nil { - - return true - } - return false -} - -func setExtraEnvVariableInDeployStep(deploySteps []*bean3.StepObject, extraEnvVariables map[string]string, webhookAndCiData *gitSensorClient.WebhookAndCiData) { - for _, deployStep := range deploySteps { - for variableKey, variableValue := range extraEnvVariables { - if isExtraVariableDynamic(variableKey, webhookAndCiData) && deployStep.StepType == "INLINE" { - extraInputVar := &bean3.VariableObject{ - Name: variableKey, - Format: "STRING", - Value: variableValue, - VariableType: bean3.VARIABLE_TYPE_REF_GLOBAL, - ReferenceVariableName: variableKey, - } - deployStep.InputVars = append(deployStep.InputVars, extraInputVar) - } - } - } -} -func (impl *WorkflowDagExecutorImpl) buildWFRequest(runner *pipelineConfig.CdWorkflowRunner, cdWf *pipelineConfig.CdWorkflow, cdPipeline *pipelineConfig.Pipeline, triggeredBy int32) (*types.WorkflowRequest, error) { - cdWorkflowConfig, err := impl.cdWorkflowRepository.FindConfigByPipelineId(cdPipeline.Id) - if err != nil && !util.IsErrNoRows(err) { - return nil, err - } - - workflowExecutor := runner.ExecutorType - - artifact, err := impl.ciArtifactRepository.Get(cdWf.CiArtifactId) - if err != nil { - return nil, err - } - // Migration of deprecated DataSource Type - if artifact.IsMigrationRequired() { - migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(artifact.Id) - if migrationErr != nil { - impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", artifact.Id) - } - } - ciMaterialInfo, err := repository.GetCiMaterialInfo(artifact.MaterialInfo, artifact.DataSource) - if err != nil { - impl.logger.Errorw("parsing error", "err", err) - return nil, err - } - - var ciProjectDetails []bean3.CiProjectDetails - var ciPipeline *pipelineConfig.CiPipeline - if cdPipeline.CiPipelineId > 0 { - ciPipeline, err = impl.ciPipelineRepository.FindById(cdPipeline.CiPipelineId) - if err != nil && !util.IsErrNoRows(err) { - impl.logger.Errorw("cannot find ciPipelineRequest", "err", err) - return nil, err - } - - for _, m := range ciPipeline.CiPipelineMaterials { - // git material should be active in this case - if m == nil || m.GitMaterial == nil || !m.GitMaterial.Active { - continue - } - var ciMaterialCurrent repository.CiMaterialInfo - for _, ciMaterial := range ciMaterialInfo { - if ciMaterial.Material.GitConfiguration.URL == m.GitMaterial.Url { - ciMaterialCurrent = ciMaterial - break - } - } - gitMaterial, err := impl.materialRepository.FindById(m.GitMaterialId) - if err != nil && !util.IsErrNoRows(err) { - impl.logger.Errorw("could not fetch git materials", "err", err) - return nil, err - } - - ciProjectDetail := bean3.CiProjectDetails{ - GitRepository: ciMaterialCurrent.Material.GitConfiguration.URL, - MaterialName: gitMaterial.Name, - CheckoutPath: gitMaterial.CheckoutPath, - FetchSubmodules: gitMaterial.FetchSubmodules, - SourceType: m.Type, - SourceValue: m.Value, - Type: string(m.Type), - GitOptions: bean3.GitOptions{ - UserName: gitMaterial.GitProvider.UserName, - Password: gitMaterial.GitProvider.Password, - SshPrivateKey: gitMaterial.GitProvider.SshPrivateKey, - AccessToken: gitMaterial.GitProvider.AccessToken, - AuthMode: gitMaterial.GitProvider.AuthMode, - }, - } - - if len(ciMaterialCurrent.Modifications) > 0 { - ciProjectDetail.CommitHash = ciMaterialCurrent.Modifications[0].Revision - ciProjectDetail.Author = ciMaterialCurrent.Modifications[0].Author - ciProjectDetail.GitTag = ciMaterialCurrent.Modifications[0].Tag - ciProjectDetail.Message = ciMaterialCurrent.Modifications[0].Message - commitTime, err := convert(ciMaterialCurrent.Modifications[0].ModifiedTime) - if err != nil { - return nil, err - } - ciProjectDetail.CommitTime = commitTime.Format(bean2.LayoutRFC3339) - } else if ciPipeline.PipelineType == bean3.CI_JOB { - // This has been done to resolve unmarshalling issue in ci-runner, in case of no commit time(eg- polling container images) - ciProjectDetail.CommitTime = time.Time{}.Format(bean2.LayoutRFC3339) - } else { - impl.logger.Debugw("devtronbug#1062", ciPipeline.Id, cdPipeline.Id) - return nil, fmt.Errorf("modifications not found for %d", ciPipeline.Id) - } - - // set webhook data - if m.Type == pipelineConfig.SOURCE_TYPE_WEBHOOK && len(ciMaterialCurrent.Modifications) > 0 { - webhookData := ciMaterialCurrent.Modifications[0].WebhookData - ciProjectDetail.WebhookData = pipelineConfig.WebhookData{ - Id: webhookData.Id, - EventActionType: webhookData.EventActionType, - Data: webhookData.Data, - } - } - - ciProjectDetails = append(ciProjectDetails, ciProjectDetail) - } - } - var stageYaml string - var deployStageWfr pipelineConfig.CdWorkflowRunner - var deployStageTriggeredByUserEmail string - var pipelineReleaseCounter int - var preDeploySteps []*bean3.StepObject - var postDeploySteps []*bean3.StepObject - var refPluginsData []*bean3.RefPluginObject - //if pipeline_stage_steps present for pre-CD or post-CD then no need to add stageYaml to cdWorkflowRequest in that - //case add PreDeploySteps and PostDeploySteps to cdWorkflowRequest, this is done for backward compatibility - pipelineStage, err := impl.getPipelineStage(cdPipeline.Id, runner.WorkflowType.WorkflowTypeToStageType()) - if err != nil { - return nil, err - } - env, err := impl.envRepository.FindById(cdPipeline.EnvironmentId) - if err != nil { - impl.logger.Errorw("error in getting environment by id", "err", err) - return nil, err - } - - //Scope will pick the environment of CD pipeline irrespective of in-cluster mode, - //since user sees the environment of the CD pipeline - scope := resourceQualifiers.Scope{ - AppId: cdPipeline.App.Id, - EnvId: env.Id, - ClusterId: env.ClusterId, - SystemMetadata: &resourceQualifiers.SystemMetadata{ - EnvironmentName: env.Name, - ClusterName: env.Cluster.ClusterName, - Namespace: env.Namespace, - Image: artifact.Image, - ImageTag: util3.GetImageTagFromImage(artifact.Image), - }, - } - if pipelineStage != nil { - var variableSnapshot map[string]string - if runner.WorkflowType == bean.CD_WORKFLOW_TYPE_PRE { - //preDeploySteps, _, refPluginsData, err = impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(cdPipeline.Id, cdStage) - prePostAndRefPluginResponse, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(cdPipeline.Id, preCdStage, scope) - if err != nil { - impl.logger.Errorw("error in getting pre, post & refPlugin steps data for wf request", "err", err, "cdPipelineId", cdPipeline.Id) - return nil, err - } - preDeploySteps = prePostAndRefPluginResponse.PreStageSteps - refPluginsData = prePostAndRefPluginResponse.RefPluginData - variableSnapshot = prePostAndRefPluginResponse.VariableSnapshot - } else if runner.WorkflowType == bean.CD_WORKFLOW_TYPE_POST { - //_, postDeploySteps, refPluginsData, err = impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(cdPipeline.Id, cdStage) - prePostAndRefPluginResponse, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(cdPipeline.Id, postCdStage, scope) - if err != nil { - impl.logger.Errorw("error in getting pre, post & refPlugin steps data for wf request", "err", err, "cdPipelineId", cdPipeline.Id) - return nil, err - } - postDeploySteps = prePostAndRefPluginResponse.PostStageSteps - refPluginsData = prePostAndRefPluginResponse.RefPluginData - variableSnapshot = prePostAndRefPluginResponse.VariableSnapshot - deployStageWfr, deployStageTriggeredByUserEmail, pipelineReleaseCounter, err = impl.getDeployStageDetails(cdPipeline.Id) - if err != nil { - impl.logger.Errorw("error in getting deployStageWfr, deployStageTriggeredByUser and pipelineReleaseCounter wf request", "err", err, "cdPipelineId", cdPipeline.Id) - return nil, err - } - } else { - return nil, fmt.Errorf("unsupported workflow triggerd") - } - - //Save Scoped VariableSnapshot - var variableSnapshotHistories = util4.GetBeansPtr( - repository5.GetSnapshotBean(runner.Id, repository5.HistoryReferenceTypeCDWORKFLOWRUNNER, variableSnapshot)) - if len(variableSnapshotHistories) > 0 { - err = impl.scopedVariableManager.SaveVariableHistoriesForTrigger(variableSnapshotHistories, runner.TriggeredBy) - if err != nil { - impl.logger.Errorf("Not able to save variable snapshot for CD trigger %s %d %s", err, runner.Id, variableSnapshot) - } - } - } else { - //in this case no plugin script is not present for this cdPipeline hence going with attaching preStage or postStage config - if runner.WorkflowType == bean.CD_WORKFLOW_TYPE_PRE { - stageYaml = cdPipeline.PreStageConfig - } else if runner.WorkflowType == bean.CD_WORKFLOW_TYPE_POST { - stageYaml = cdPipeline.PostStageConfig - deployStageWfr, deployStageTriggeredByUserEmail, pipelineReleaseCounter, err = impl.getDeployStageDetails(cdPipeline.Id) - if err != nil { - impl.logger.Errorw("error in getting deployStageWfr, deployStageTriggeredByUser and pipelineReleaseCounter wf request", "err", err, "cdPipelineId", cdPipeline.Id) - return nil, err - } - - } else { - return nil, fmt.Errorf("unsupported workflow triggerd") - } - } - - digestConfigurationRequest := imageDigestPolicy.DigestPolicyConfigurationRequest{PipelineId: cdPipeline.Id} - digestPolicyConfigurations, err := impl.imageDigestPolicyService.GetDigestPolicyConfigurations(digestConfigurationRequest) - if err != nil { - impl.logger.Errorw("error in checking if isImageDigestPolicyConfiguredForPipeline", "err", err, "pipelineId", cdPipeline.Id) - return nil, err - } - image := artifact.Image - if digestPolicyConfigurations.UseDigestForTrigger() { - image = ReplaceImageTagWithDigest(image, artifact.ImageDigest) - } - cdStageWorkflowRequest := &types.WorkflowRequest{ - EnvironmentId: cdPipeline.EnvironmentId, - AppId: cdPipeline.AppId, - WorkflowId: cdWf.Id, - WorkflowRunnerId: runner.Id, - WorkflowNamePrefix: strconv.Itoa(runner.Id) + "-" + runner.Name, - WorkflowPrefixForLog: strconv.Itoa(cdWf.Id) + string(runner.WorkflowType) + "-" + runner.Name, - CdImage: impl.config.GetDefaultImage(), - CdPipelineId: cdWf.PipelineId, - TriggeredBy: triggeredBy, - StageYaml: stageYaml, - CiProjectDetails: ciProjectDetails, - Namespace: runner.Namespace, - ActiveDeadlineSeconds: impl.config.GetDefaultTimeout(), - CiArtifactDTO: types.CiArtifactDTO{ - Id: artifact.Id, - PipelineId: artifact.PipelineId, - Image: artifact.Image, - ImageDigest: artifact.ImageDigest, - MaterialInfo: artifact.MaterialInfo, - DataSource: artifact.DataSource, - WorkflowId: artifact.WorkflowId, - }, - OrchestratorHost: impl.config.OrchestratorHost, - OrchestratorToken: impl.config.OrchestratorToken, - CloudProvider: impl.config.CloudProvider, - WorkflowExecutor: workflowExecutor, - RefPlugins: refPluginsData, - Scope: scope, - } - - extraEnvVariables := make(map[string]string) - if env != nil { - extraEnvVariables[plugin.CD_PIPELINE_ENV_NAME_KEY] = env.Name - if env.Cluster != nil { - extraEnvVariables[plugin.CD_PIPELINE_CLUSTER_NAME_KEY] = env.Cluster.ClusterName - } - } - ciWf, err := impl.ciWorkflowRepository.FindLastTriggeredWorkflowByArtifactId(artifact.Id) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in getting ciWf by artifactId", "err", err, "artifactId", artifact.Id) - return nil, err - } - var webhookAndCiData *gitSensorClient.WebhookAndCiData - if ciWf != nil && ciWf.GitTriggers != nil { - i := 1 - var gitCommitEnvVariables []types.GitMetadata - - for ciPipelineMaterialId, gitTrigger := range ciWf.GitTriggers { - extraEnvVariables[fmt.Sprintf("%s_%d", GIT_COMMIT_HASH_PREFIX, i)] = gitTrigger.Commit - extraEnvVariables[fmt.Sprintf("%s_%d", GIT_SOURCE_TYPE_PREFIX, i)] = string(gitTrigger.CiConfigureSourceType) - extraEnvVariables[fmt.Sprintf("%s_%d", GIT_SOURCE_VALUE_PREFIX, i)] = gitTrigger.CiConfigureSourceValue - - gitCommitEnvVariables = append(gitCommitEnvVariables, types.GitMetadata{ - GitCommitHash: gitTrigger.Commit, - GitSourceType: string(gitTrigger.CiConfigureSourceType), - GitSourceValue: gitTrigger.CiConfigureSourceValue, - }) - - // CODE-BLOCK starts - store extra environment variables if webhook - if gitTrigger.CiConfigureSourceType == pipelineConfig.SOURCE_TYPE_WEBHOOK { - webhookDataId := gitTrigger.WebhookData.Id - if webhookDataId > 0 { - webhookDataRequest := &gitSensorClient.WebhookDataRequest{ - Id: webhookDataId, - CiPipelineMaterialId: ciPipelineMaterialId, - } - webhookAndCiData, err = impl.gitSensorGrpcClient.GetWebhookData(context.Background(), webhookDataRequest) - if err != nil { - impl.logger.Errorw("err while getting webhook data from git-sensor", "err", err, "webhookDataRequest", webhookDataRequest) - return nil, err - } - if webhookAndCiData != nil { - for extEnvVariableKey, extEnvVariableVal := range webhookAndCiData.ExtraEnvironmentVariables { - extraEnvVariables[extEnvVariableKey] = extEnvVariableVal - } - } - } - } - // CODE_BLOCK ends - - i++ - } - gitMetadata, err := json.Marshal(&gitCommitEnvVariables) - if err != nil { - impl.logger.Errorw("err while marshaling git metdata", "err", err) - return nil, err - } - extraEnvVariables[plugin.GIT_METADATA] = string(gitMetadata) - - extraEnvVariables[GIT_SOURCE_COUNT] = strconv.Itoa(len(ciWf.GitTriggers)) - } - - childCdIds, err := impl.appWorkflowRepository.FindChildCDIdsByParentCDPipelineId(cdPipeline.Id) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in getting child cdPipelineIds by parent cdPipelineId", "err", err, "parent cdPipelineId", cdPipeline.Id) - return nil, err - } - if len(childCdIds) > 0 { - childPipelines, err := impl.pipelineRepository.FindByIdsIn(childCdIds) - if err != nil { - impl.logger.Errorw("error in getting pipelines by ids", "err", err, "ids", childCdIds) - return nil, err - } - var childCdEnvVariables []types.ChildCdMetadata - for i, childPipeline := range childPipelines { - extraEnvVariables[fmt.Sprintf("%s_%d", CHILD_CD_ENV_NAME_PREFIX, i+1)] = childPipeline.Environment.Name - extraEnvVariables[fmt.Sprintf("%s_%d", CHILD_CD_CLUSTER_NAME_PREFIX, i+1)] = childPipeline.Environment.Cluster.ClusterName - - childCdEnvVariables = append(childCdEnvVariables, types.ChildCdMetadata{ - ChildCdEnvName: childPipeline.Environment.Name, - ChildCdClusterName: childPipeline.Environment.Cluster.ClusterName, - }) - } - childCdEnvVariablesMetadata, err := json.Marshal(&childCdEnvVariables) - if err != nil { - impl.logger.Errorw("err while marshaling childCdEnvVariables", "err", err) - return nil, err - } - extraEnvVariables[plugin.CHILD_CD_METADATA] = string(childCdEnvVariablesMetadata) - - extraEnvVariables[CHILD_CD_COUNT] = strconv.Itoa(len(childPipelines)) - } - if ciPipeline != nil && ciPipeline.Id > 0 { - extraEnvVariables["APP_NAME"] = ciPipeline.App.AppName - cdStageWorkflowRequest.DockerUsername = ciPipeline.CiTemplate.DockerRegistry.Username - cdStageWorkflowRequest.DockerPassword = ciPipeline.CiTemplate.DockerRegistry.Password - cdStageWorkflowRequest.AwsRegion = ciPipeline.CiTemplate.DockerRegistry.AWSRegion - cdStageWorkflowRequest.DockerConnection = ciPipeline.CiTemplate.DockerRegistry.Connection - cdStageWorkflowRequest.DockerCert = ciPipeline.CiTemplate.DockerRegistry.Cert - cdStageWorkflowRequest.AccessKey = ciPipeline.CiTemplate.DockerRegistry.AWSAccessKeyId - cdStageWorkflowRequest.SecretKey = ciPipeline.CiTemplate.DockerRegistry.AWSSecretAccessKey - cdStageWorkflowRequest.DockerRegistryType = string(ciPipeline.CiTemplate.DockerRegistry.RegistryType) - cdStageWorkflowRequest.DockerRegistryURL = ciPipeline.CiTemplate.DockerRegistry.RegistryURL - cdStageWorkflowRequest.DockerRegistryId = ciPipeline.CiTemplate.DockerRegistry.Id - cdStageWorkflowRequest.CiPipelineType = ciPipeline.PipelineType - } else if cdPipeline.AppId > 0 { - ciTemplate, err := impl.CiTemplateRepository.FindByAppId(cdPipeline.AppId) - if err != nil { - return nil, err - } - extraEnvVariables["APP_NAME"] = ciTemplate.App.AppName - cdStageWorkflowRequest.DockerUsername = ciTemplate.DockerRegistry.Username - cdStageWorkflowRequest.DockerPassword = ciTemplate.DockerRegistry.Password - cdStageWorkflowRequest.AwsRegion = ciTemplate.DockerRegistry.AWSRegion - cdStageWorkflowRequest.DockerConnection = ciTemplate.DockerRegistry.Connection - cdStageWorkflowRequest.DockerCert = ciTemplate.DockerRegistry.Cert - cdStageWorkflowRequest.AccessKey = ciTemplate.DockerRegistry.AWSAccessKeyId - cdStageWorkflowRequest.SecretKey = ciTemplate.DockerRegistry.AWSSecretAccessKey - cdStageWorkflowRequest.DockerRegistryType = string(ciTemplate.DockerRegistry.RegistryType) - cdStageWorkflowRequest.DockerRegistryURL = ciTemplate.DockerRegistry.RegistryURL - appLabels, err := impl.appLabelRepository.FindAllByAppId(cdPipeline.AppId) - cdStageWorkflowRequest.DockerRegistryId = ciTemplate.DockerRegistry.Id - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in getting labels by appId", "err", err, "appId", cdPipeline.AppId) - return nil, err - } - var appLabelEnvVariables []types.AppLabelMetadata - for i, appLabel := range appLabels { - extraEnvVariables[fmt.Sprintf("%s_%d", APP_LABEL_KEY_PREFIX, i+1)] = appLabel.Key - extraEnvVariables[fmt.Sprintf("%s_%d", APP_LABEL_VALUE_PREFIX, i+1)] = appLabel.Value - appLabelEnvVariables = append(appLabelEnvVariables, types.AppLabelMetadata{ - AppLabelKey: appLabel.Key, - AppLabelValue: appLabel.Value, - }) - } - if len(appLabels) > 0 { - extraEnvVariables[APP_LABEL_COUNT] = strconv.Itoa(len(appLabels)) - appLabelEnvVariablesMetadata, err := json.Marshal(&appLabelEnvVariables) - if err != nil { - impl.logger.Errorw("err while marshaling appLabelEnvVariables", "err", err) - return nil, err - } - extraEnvVariables[plugin.APP_LABEL_METADATA] = string(appLabelEnvVariablesMetadata) - - } - } - cdStageWorkflowRequest.ExtraEnvironmentVariables = extraEnvVariables - cdStageWorkflowRequest.DeploymentTriggerTime = deployStageWfr.StartedOn - cdStageWorkflowRequest.DeploymentTriggeredBy = deployStageTriggeredByUserEmail - - if pipelineReleaseCounter > 0 { - cdStageWorkflowRequest.DeploymentReleaseCounter = pipelineReleaseCounter - } - if cdWorkflowConfig.CdCacheRegion == "" { - cdWorkflowConfig.CdCacheRegion = impl.config.GetDefaultCdLogsBucketRegion() - } - - if runner.WorkflowType == bean.CD_WORKFLOW_TYPE_PRE { - //populate input variables of steps with extra env variables - setExtraEnvVariableInDeployStep(preDeploySteps, extraEnvVariables, webhookAndCiData) - cdStageWorkflowRequest.PrePostDeploySteps = preDeploySteps - } else if runner.WorkflowType == bean.CD_WORKFLOW_TYPE_POST { - setExtraEnvVariableInDeployStep(postDeploySteps, extraEnvVariables, webhookAndCiData) - cdStageWorkflowRequest.PrePostDeploySteps = postDeploySteps - } - cdStageWorkflowRequest.BlobStorageConfigured = runner.BlobStorageEnabled - switch cdStageWorkflowRequest.CloudProvider { - case types.BLOB_STORAGE_S3: - //No AccessKey is used for uploading artifacts, instead IAM based auth is used - cdStageWorkflowRequest.CdCacheRegion = cdWorkflowConfig.CdCacheRegion - cdStageWorkflowRequest.CdCacheLocation = cdWorkflowConfig.CdCacheBucket - cdStageWorkflowRequest.ArtifactLocation, cdStageWorkflowRequest.CiArtifactBucket, cdStageWorkflowRequest.CiArtifactFileName = impl.buildArtifactLocationForS3(cdWorkflowConfig, cdWf, runner) - cdStageWorkflowRequest.BlobStorageS3Config = &blob_storage.BlobStorageS3Config{ - AccessKey: impl.config.BlobStorageS3AccessKey, - Passkey: impl.config.BlobStorageS3SecretKey, - EndpointUrl: impl.config.BlobStorageS3Endpoint, - IsInSecure: impl.config.BlobStorageS3EndpointInsecure, - CiCacheBucketName: cdWorkflowConfig.CdCacheBucket, - CiCacheRegion: cdWorkflowConfig.CdCacheRegion, - CiCacheBucketVersioning: impl.config.BlobStorageS3BucketVersioned, - CiArtifactBucketName: cdStageWorkflowRequest.CiArtifactBucket, - CiArtifactRegion: cdWorkflowConfig.CdCacheRegion, - CiArtifactBucketVersioning: impl.config.BlobStorageS3BucketVersioned, - CiLogBucketName: impl.config.GetDefaultBuildLogsBucket(), - CiLogRegion: impl.config.GetDefaultCdLogsBucketRegion(), - CiLogBucketVersioning: impl.config.BlobStorageS3BucketVersioned, - } - case types.BLOB_STORAGE_GCP: - cdStageWorkflowRequest.GcpBlobConfig = &blob_storage.GcpBlobConfig{ - CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, - ArtifactBucketName: impl.config.GetDefaultBuildLogsBucket(), - LogBucketName: impl.config.GetDefaultBuildLogsBucket(), - } - cdStageWorkflowRequest.ArtifactLocation = impl.buildDefaultArtifactLocation(cdWorkflowConfig, cdWf, runner) - cdStageWorkflowRequest.CiArtifactFileName = cdStageWorkflowRequest.ArtifactLocation - case types.BLOB_STORAGE_AZURE: - cdStageWorkflowRequest.AzureBlobConfig = &blob_storage.AzureBlobConfig{ - Enabled: true, - AccountName: impl.config.AzureAccountName, - BlobContainerCiCache: impl.config.AzureBlobContainerCiCache, - AccountKey: impl.config.AzureAccountKey, - BlobContainerCiLog: impl.config.AzureBlobContainerCiLog, - BlobContainerArtifact: impl.config.AzureBlobContainerCiLog, - } - cdStageWorkflowRequest.BlobStorageS3Config = &blob_storage.BlobStorageS3Config{ - EndpointUrl: impl.config.AzureGatewayUrl, - IsInSecure: impl.config.AzureGatewayConnectionInsecure, - CiLogBucketName: impl.config.AzureBlobContainerCiLog, - CiLogRegion: "", - AccessKey: impl.config.AzureAccountName, - } - cdStageWorkflowRequest.ArtifactLocation = impl.buildDefaultArtifactLocation(cdWorkflowConfig, cdWf, runner) - cdStageWorkflowRequest.CiArtifactFileName = cdStageWorkflowRequest.ArtifactLocation - default: - if impl.config.BlobStorageEnabled { - return nil, fmt.Errorf("blob storage %s not supported", cdStageWorkflowRequest.CloudProvider) - } - } - cdStageWorkflowRequest.DefaultAddressPoolBaseCidr = impl.config.GetDefaultAddressPoolBaseCidr() - cdStageWorkflowRequest.DefaultAddressPoolSize = impl.config.GetDefaultAddressPoolSize() - return cdStageWorkflowRequest, nil -} - -func ReplaceImageTagWithDigest(image, digest string) string { - imageWithoutTag := strings.Split(image, ":")[0] - imageWithDigest := fmt.Sprintf("%s@%s", imageWithoutTag, digest) - return imageWithDigest -} - -func (impl *WorkflowDagExecutorImpl) buildDefaultArtifactLocation(cdWorkflowConfig *pipelineConfig.CdWorkflowConfig, savedWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner) string { - cdArtifactLocationFormat := cdWorkflowConfig.CdArtifactLocationFormat - if cdArtifactLocationFormat == "" { - cdArtifactLocationFormat = impl.config.GetArtifactLocationFormat() - } - ArtifactLocation := fmt.Sprintf("%s/"+cdArtifactLocationFormat, impl.config.GetDefaultArtifactKeyPrefix(), savedWf.Id, runner.Id) - return ArtifactLocation -} - -func (impl *WorkflowDagExecutorImpl) HandleDeploymentSuccessEvent(triggerContext TriggerContext, pipelineOverride *chartConfig.PipelineOverride) error { - if pipelineOverride == nil { - return fmt.Errorf("invalid request, pipeline override not found") - } - cdWorkflow, err := impl.cdWorkflowRepository.FindById(pipelineOverride.CdWorkflowId) - if err != nil { - impl.logger.Errorw("error in fetching cd workflow by id", "pipelineOverride", pipelineOverride) - return err - } - - postStage, err := impl.getPipelineStage(pipelineOverride.PipelineId, repository4.PIPELINE_STAGE_TYPE_POST_CD) - if err != nil { - return err - } - - var triggeredByUser int32 = 1 - //handle corrupt data (https://github.com/devtron-labs/devtron/issues/3826) - err, deleted := impl.deleteCorruptedPipelineStage(postStage, triggeredByUser) - if err != nil { - impl.logger.Errorw("error in deleteCorruptedPipelineStage ", "err", err, "preStage", postStage, "triggeredBy", triggeredByUser) - return err - } - - if len(pipelineOverride.Pipeline.PostStageConfig) > 0 || (postStage != nil && !deleted) { - if pipelineOverride.Pipeline.PostTriggerType == pipelineConfig.TRIGGER_TYPE_AUTOMATIC && - pipelineOverride.DeploymentType != models.DEPLOYMENTTYPE_STOP && - pipelineOverride.DeploymentType != models.DEPLOYMENTTYPE_START { - - triggerRequest := TriggerRequest{ - CdWf: cdWorkflow, - Pipeline: pipelineOverride.Pipeline, - TriggeredBy: triggeredByUser, - TriggerContext: triggerContext, - RefCdWorkflowRunnerId: 0, - } - triggerRequest.TriggerContext.Context = context.Background() - err = impl.TriggerPostStage(triggerRequest) - if err != nil { - impl.logger.Errorw("error in triggering post stage after successful deployment event", "err", err, "cdWorkflow", cdWorkflow) - return err - } - } - } else { - // to trigger next pre/cd, if any - // finding children cd by pipeline id - err = impl.HandlePostStageSuccessEvent(triggerContext, cdWorkflow.Id, pipelineOverride.PipelineId, 1, nil) - if err != nil { - impl.logger.Errorw("error in triggering children cd after successful deployment event", "parentCdPipelineId", pipelineOverride.PipelineId) - return err - } - } - return nil -} - -func (impl *WorkflowDagExecutorImpl) HandlePostStageSuccessEvent(triggerContext TriggerContext, cdWorkflowId int, cdPipelineId int, triggeredBy int32, pluginRegistryImageDetails map[string][]string) error { - // finding children cd by pipeline id - cdPipelinesMapping, err := impl.appWorkflowRepository.FindWFCDMappingByParentCDPipelineId(cdPipelineId) - if err != nil { - impl.logger.Errorw("error in getting mapping of cd pipelines by parent cd pipeline id", "err", err, "parentCdPipelineId", cdPipelineId) - return err - } - ciArtifact, err := impl.ciArtifactRepository.GetArtifactByCdWorkflowId(cdWorkflowId) - if err != nil { - impl.logger.Errorw("error in finding artifact by cd workflow id", "err", err, "cdWorkflowId", cdWorkflowId) - return err - } - if len(pluginRegistryImageDetails) > 0 { - PostCDArtifacts, err := impl.commonArtifactService.SavePluginArtifacts(ciArtifact, pluginRegistryImageDetails, cdPipelineId, repository.POST_CD, triggeredBy) - if err != nil { - impl.logger.Errorw("error in saving plugin artifacts", "err", err) - return err - } - if len(PostCDArtifacts) > 0 { - ciArtifact = PostCDArtifacts[0] - } - } - for _, cdPipelineMapping := range cdPipelinesMapping { - //find pipeline by cdPipeline ID - pipeline, err := impl.pipelineRepository.FindById(cdPipelineMapping.ComponentId) - if err != nil { - impl.logger.Errorw("error in getting cd pipeline by id", "err", err, "pipelineId", cdPipelineMapping.ComponentId) - return err - } - //finding ci artifact by ciPipelineID and pipelineId - //TODO : confirm values for applyAuth, async & triggeredBy - - triggerRequest := TriggerRequest{ - CdWf: nil, - Pipeline: pipeline, - Artifact: ciArtifact, - TriggeredBy: triggeredBy, - TriggerContext: triggerContext, - } - - err = impl.triggerIfAutoStageCdPipeline(triggerRequest) - if err != nil { - impl.logger.Errorw("error in triggering cd pipeline after successful post stage", "err", err, "pipelineId", pipeline.Id) - return err - } - } - return nil -} - -// Only used for auto trigger -func (impl *WorkflowDagExecutorImpl) TriggerDeployment(request TriggerRequest) error { - //in case of manual trigger auth is already applied and for auto triggers there is no need for auth check here - triggeredBy := request.TriggeredBy - pipeline := request.Pipeline - artifact := request.Artifact - - //setting triggeredAt variable to have consistent data for various audit log places in db for deployment time - triggeredAt := time.Now() - cdWf := request.CdWf - - if cdWf == nil || (cdWf != nil && cdWf.CiArtifactId != artifact.Id) { - // cdWf != nil && cdWf.CiArtifactId != artifact.Id for auto trigger case when deployment is triggered with image generated by plugin - cdWf = &pipelineConfig.CdWorkflow{ - CiArtifactId: artifact.Id, - PipelineId: pipeline.Id, - AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: 1, UpdatedOn: triggeredAt, UpdatedBy: 1}, - } - err := impl.cdWorkflowRepository.SaveWorkFlow(context.Background(), cdWf) - if err != nil { - return err - } - } - - runner := &pipelineConfig.CdWorkflowRunner{ - Name: pipeline.Name, - WorkflowType: bean.CD_WORKFLOW_TYPE_DEPLOY, - ExecutorType: pipelineConfig.WORKFLOW_EXECUTOR_TYPE_SYSTEM, - Status: pipelineConfig.WorkflowInitiated, // deployment Initiated for auto trigger - TriggeredBy: 1, - StartedOn: triggeredAt, - Namespace: impl.config.GetDefaultNamespace(), - CdWorkflowId: cdWf.Id, - AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: triggeredBy, UpdatedOn: triggeredAt, UpdatedBy: triggeredBy}, - ReferenceId: request.TriggerContext.ReferenceId, - } - savedWfr, err := impl.cdWorkflowRepository.SaveWorkFlowRunner(runner) - if err != nil { - return err - } - runner.CdWorkflow = &pipelineConfig.CdWorkflow{ - Pipeline: pipeline, - } - // creating cd pipeline status timeline for deployment initialisation - timeline := &pipelineConfig.PipelineStatusTimeline{ - CdWorkflowRunnerId: runner.Id, - Status: pipelineConfig.TIMELINE_STATUS_DEPLOYMENT_INITIATED, - StatusDetail: "Deployment initiated successfully.", - StatusTime: time.Now(), - AuditLog: sql.AuditLog{ - CreatedBy: 1, - CreatedOn: time.Now(), - UpdatedBy: 1, - UpdatedOn: time.Now(), - }, - } - isAppStore := false - err = impl.pipelineStatusTimelineService.SaveTimeline(timeline, nil, isAppStore) - if err != nil { - impl.logger.Errorw("error in creating timeline status for deployment initiation", "err", err, "timeline", timeline) - } - //checking vulnerability for deploying image - isVulnerable := false - if len(artifact.ImageDigest) > 0 { - var cveStores []*security.CveStore - imageScanResult, err := impl.scanResultRepository.FindByImageDigest(artifact.ImageDigest) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error fetching image digest", "digest", artifact.ImageDigest, "err", err) - return err - } - for _, item := range imageScanResult { - cveStores = append(cveStores, &item.CveStore) - } - env, err := impl.envRepository.FindById(pipeline.EnvironmentId) - if err != nil { - impl.logger.Errorw("error while fetching env", "err", err) - return err - } - blockCveList, err := impl.cvePolicyRepository.GetBlockedCVEList(cveStores, env.ClusterId, pipeline.EnvironmentId, pipeline.AppId, false) - if err != nil { - impl.logger.Errorw("error while fetching blocked cve list", "err", err) - return err - } - if len(blockCveList) > 0 { - isVulnerable = true - } - } - if isVulnerable == true { - if err = impl.MarkCurrentDeploymentFailed(runner, errors.New(pipelineConfig.FOUND_VULNERABILITY), triggeredBy); err != nil { - impl.logger.Errorw("error while updating current runner status to failed, TriggerDeployment", "wfrId", runner.Id, "err", err) - } - return nil - } - - releaseErr := impl.TriggerCD(artifact, cdWf.Id, savedWfr.Id, pipeline, triggeredAt) - // if releaseErr found, then the mark current deployment Failed and return - if releaseErr != nil { - err := impl.MarkCurrentDeploymentFailed(runner, releaseErr, triggeredBy) - if err != nil { - impl.logger.Errorw("error while updating current runner status to failed, updatePreviousDeploymentStatus", "cdWfr", runner.Id, "err", err) - } - return releaseErr - } - //skip updatePreviousDeploymentStatus if Async Install is enabled; handled inside SubscribeDevtronAsyncHelmInstallRequest - if !impl.cdTriggerService.IsDevtronAsyncInstallModeEnabled(pipeline.DeploymentAppType) { - err1 := impl.updatePreviousDeploymentStatus(runner, pipeline.Id, triggeredAt, triggeredBy) - if err1 != nil { - impl.logger.Errorw("error while update previous cd workflow runners", "err", err, "runner", runner, "pipelineId", pipeline.Id) - return err1 - } - } - return nil -} - -func (impl *WorkflowDagExecutorImpl) updatePreviousDeploymentStatus(currentRunner *pipelineConfig.CdWorkflowRunner, pipelineId int, triggeredAt time.Time, triggeredBy int32) error { - // Initiating DB transaction - dbConnection := impl.cdWorkflowRepository.GetConnection() - tx, err := dbConnection.Begin() - if err != nil { - impl.logger.Errorw("error on update status, txn begin failed", "err", err) - return err - } - // Rollback tx on error. - defer tx.Rollback() - - //update [n,n-1] statuses as failed if not terminal - terminalStatus := []string{string(health.HealthStatusHealthy), pipelineConfig.WorkflowAborted, pipelineConfig.WorkflowFailed, pipelineConfig.WorkflowSucceeded} - previousNonTerminalRunners, err := impl.cdWorkflowRepository.FindPreviousCdWfRunnerByStatus(pipelineId, currentRunner.Id, terminalStatus) - if err != nil { - impl.logger.Errorw("error fetching previous wf runner, updating cd wf runner status,", "err", err, "currentRunner", currentRunner) - return err - } else if len(previousNonTerminalRunners) == 0 { - impl.logger.Errorw("no previous runner found in updating cd wf runner status,", "err", err, "currentRunner", currentRunner) - return nil - } - - var timelines []*pipelineConfig.PipelineStatusTimeline - for _, previousRunner := range previousNonTerminalRunners { - if previousRunner.Status == string(health.HealthStatusHealthy) || - previousRunner.Status == pipelineConfig.WorkflowSucceeded || - previousRunner.Status == pipelineConfig.WorkflowAborted || - previousRunner.Status == pipelineConfig.WorkflowFailed { - //terminal status return - impl.logger.Infow("skip updating cd wf runner status as previous runner status is", "status", previousRunner.Status) - continue - } - impl.logger.Infow("updating cd wf runner status as previous runner status is", "status", previousRunner.Status) - previousRunner.FinishedOn = triggeredAt - previousRunner.Message = pipelineConfig.NEW_DEPLOYMENT_INITIATED - previousRunner.Status = pipelineConfig.WorkflowFailed - previousRunner.UpdatedOn = time.Now() - previousRunner.UpdatedBy = triggeredBy - timeline := &pipelineConfig.PipelineStatusTimeline{ - CdWorkflowRunnerId: previousRunner.Id, - Status: pipelineConfig.TIMELINE_STATUS_DEPLOYMENT_SUPERSEDED, - StatusDetail: "This deployment is superseded.", - StatusTime: time.Now(), - AuditLog: sql.AuditLog{ - CreatedBy: 1, - CreatedOn: time.Now(), - UpdatedBy: 1, - UpdatedOn: time.Now(), - }, - } - timelines = append(timelines, timeline) - } - - err = impl.cdWorkflowRepository.UpdateWorkFlowRunners(previousNonTerminalRunners) - if err != nil { - impl.logger.Errorw("error updating cd wf runner status", "err", err, "previousNonTerminalRunners", previousNonTerminalRunners) - return err - } - err = impl.cdPipelineStatusTimelineRepo.SaveTimelinesWithTxn(timelines, tx) - if err != nil { - impl.logger.Errorw("error updating pipeline status timelines", "err", err, "timelines", timelines) - return err - } - //commit transaction - err = tx.Commit() - if err != nil { - impl.logger.Errorw("error in db transaction commit", "err", err) - return err - } - return nil - -} - -type RequestType string - -const START RequestType = "START" -const STOP RequestType = "STOP" - -type StopAppRequest struct { - AppId int `json:"appId" validate:"required"` - EnvironmentId int `json:"environmentId" validate:"required"` - UserId int32 `json:"userId"` - RequestType RequestType `json:"requestType" validate:"oneof=START STOP"` -} - -type StopDeploymentGroupRequest struct { - DeploymentGroupId int `json:"deploymentGroupId" validate:"required"` - UserId int32 `json:"userId"` - RequestType RequestType `json:"requestType" validate:"oneof=START STOP"` -} - -func (impl *WorkflowDagExecutorImpl) StopStartApp(triggerContext TriggerContext, stopRequest *StopAppRequest) (int, error) { - pipelines, err := impl.pipelineRepository.FindActiveByAppIdAndEnvironmentId(stopRequest.AppId, stopRequest.EnvironmentId) - if err != nil { - impl.logger.Errorw("error in fetching pipeline", "app", stopRequest.AppId, "env", stopRequest.EnvironmentId, "err", err) - return 0, err - } - if len(pipelines) == 0 { - return 0, fmt.Errorf("no pipeline found") - } - pipeline := pipelines[0] - - //find pipeline with default - var pipelineIds []int - for _, p := range pipelines { - impl.logger.Debugw("adding pipelineId", "pipelineId", p.Id) - pipelineIds = append(pipelineIds, p.Id) - //FIXME - } - wf, err := impl.cdWorkflowRepository.FindLatestCdWorkflowByPipelineId(pipelineIds) - if err != nil { - impl.logger.Errorw("error in fetching latest release", "err", err) - return 0, err - } - stopTemplate := `{"replicaCount":0,"autoscaling":{"MinReplicas":0,"MaxReplicas":0 ,"enabled": false} }` - overrideRequest := &bean.ValuesOverrideRequest{ - PipelineId: pipeline.Id, - AppId: stopRequest.AppId, - CiArtifactId: wf.CiArtifactId, - UserId: stopRequest.UserId, - CdWorkflowType: bean.CD_WORKFLOW_TYPE_DEPLOY, - } - if stopRequest.RequestType == STOP { - overrideRequest.AdditionalOverride = json.RawMessage([]byte(stopTemplate)) - overrideRequest.DeploymentType = models.DEPLOYMENTTYPE_STOP - } else if stopRequest.RequestType == START { - overrideRequest.DeploymentType = models.DEPLOYMENTTYPE_START - } else { - return 0, fmt.Errorf("unsupported operation %s", stopRequest.RequestType) - } - id, err := impl.ManualCdTrigger(triggerContext, overrideRequest) - if err != nil { - impl.logger.Errorw("error in stopping app", "err", err, "appId", stopRequest.AppId, "envId", stopRequest.EnvironmentId) - return 0, err - } - return id, err -} - -func (impl *WorkflowDagExecutorImpl) GetArtifactVulnerabilityStatus(artifact *repository.CiArtifact, cdPipeline *pipelineConfig.Pipeline, ctx context.Context) (bool, error) { - isVulnerable := false - if len(artifact.ImageDigest) > 0 { - var cveStores []*security.CveStore - _, span := otel.Tracer("orchestrator").Start(ctx, "scanResultRepository.FindByImageDigest") - imageScanResult, err := impl.scanResultRepository.FindByImageDigest(artifact.ImageDigest) - span.End() - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error fetching image digest", "digest", artifact.ImageDigest, "err", err) - return false, err - } - for _, item := range imageScanResult { - cveStores = append(cveStores, &item.CveStore) - } - _, span = otel.Tracer("orchestrator").Start(ctx, "cvePolicyRepository.GetBlockedCVEList") - if cdPipeline.Environment.ClusterId == 0 { - envDetails, err := impl.envRepository.FindById(cdPipeline.EnvironmentId) - if err != nil { - impl.logger.Errorw("error fetching cluster details by env, GetArtifactVulnerabilityStatus", "envId", cdPipeline.EnvironmentId, "err", err) - return false, err - } - cdPipeline.Environment = *envDetails - } - blockCveList, err := impl.cvePolicyRepository.GetBlockedCVEList(cveStores, cdPipeline.Environment.ClusterId, cdPipeline.EnvironmentId, cdPipeline.AppId, false) - span.End() - if err != nil { - impl.logger.Errorw("error while fetching env", "err", err) - return false, err - } - if len(blockCveList) > 0 { - isVulnerable = true - } - } - return isVulnerable, nil -} - -func (impl *WorkflowDagExecutorImpl) ManualCdTrigger(triggerContext TriggerContext, overrideRequest *bean.ValuesOverrideRequest) (int, error) { - //setting triggeredAt variable to have consistent data for various audit log places in db for deployment time - triggeredAt := time.Now() - releaseId := 0 - ctx := triggerContext.Context - var err error - _, span := otel.Tracer("orchestrator").Start(ctx, "pipelineRepository.FindById") - cdPipeline, err := impl.pipelineRepository.FindById(overrideRequest.PipelineId) - span.End() - if err != nil { - impl.logger.Errorw("manual trigger request with invalid pipelineId, ManualCdTrigger", "pipelineId", overrideRequest.PipelineId, "err", err) - return 0, err - } - impl.SetPipelineFieldsInOverrideRequest(overrideRequest, cdPipeline) - - switch overrideRequest.CdWorkflowType { - case bean.CD_WORKFLOW_TYPE_PRE: - _, span = otel.Tracer("orchestrator").Start(ctx, "ciArtifactRepository.Get") - artifact, err := impl.ciArtifactRepository.Get(overrideRequest.CiArtifactId) - span.End() - if err != nil { - impl.logger.Errorw("error in getting CiArtifact", "CiArtifactId", overrideRequest.CiArtifactId, "err", err) - return 0, err - } - // Migration of deprecated DataSource Type - if artifact.IsMigrationRequired() { - migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(artifact.Id) - if migrationErr != nil { - impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", artifact.Id) - } - } - _, span = otel.Tracer("orchestrator").Start(ctx, "TriggerPreStage") - triggerRequest := TriggerRequest{ - CdWf: nil, - Artifact: artifact, - Pipeline: cdPipeline, - TriggeredBy: overrideRequest.UserId, - ApplyAuth: false, - TriggerContext: triggerContext, - RefCdWorkflowRunnerId: 0, - } - err = impl.TriggerPreStage(triggerRequest) - span.End() - if err != nil { - impl.logger.Errorw("error in TriggerPreStage, ManualCdTrigger", "err", err) - return 0, err - } - case bean.CD_WORKFLOW_TYPE_DEPLOY: - if overrideRequest.DeploymentType == models.DEPLOYMENTTYPE_UNKNOWN { - overrideRequest.DeploymentType = models.DEPLOYMENTTYPE_DEPLOY - } - - cdWf, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(ctx, overrideRequest.CdWorkflowId, bean.CD_WORKFLOW_TYPE_PRE) - if err != nil && !util.IsErrNoRows(err) { - impl.logger.Errorw("error in getting cdWorkflow, ManualCdTrigger", "CdWorkflowId", overrideRequest.CdWorkflowId, "err", err) - return 0, err - } - - cdWorkflowId := cdWf.CdWorkflowId - if cdWf.CdWorkflowId == 0 { - cdWf := &pipelineConfig.CdWorkflow{ - CiArtifactId: overrideRequest.CiArtifactId, - PipelineId: overrideRequest.PipelineId, - AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: overrideRequest.UserId, UpdatedOn: triggeredAt, UpdatedBy: overrideRequest.UserId}, - } - err := impl.cdWorkflowRepository.SaveWorkFlow(ctx, cdWf) - if err != nil { - impl.logger.Errorw("error in creating cdWorkflow, ManualCdTrigger", "PipelineId", overrideRequest.PipelineId, "err", err) - return 0, err - } - cdWorkflowId = cdWf.Id - } - - runner := &pipelineConfig.CdWorkflowRunner{ - Name: cdPipeline.Name, - WorkflowType: bean.CD_WORKFLOW_TYPE_DEPLOY, - ExecutorType: pipelineConfig.WORKFLOW_EXECUTOR_TYPE_AWF, - Status: pipelineConfig.WorkflowInitiated, //deployment Initiated for manual trigger - TriggeredBy: overrideRequest.UserId, - StartedOn: triggeredAt, - Namespace: impl.config.GetDefaultNamespace(), - CdWorkflowId: cdWorkflowId, - AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: overrideRequest.UserId, UpdatedOn: triggeredAt, UpdatedBy: overrideRequest.UserId}, - ReferenceId: triggerContext.ReferenceId, - } - savedWfr, err := impl.cdWorkflowRepository.SaveWorkFlowRunner(runner) - overrideRequest.WfrId = savedWfr.Id - if err != nil { - impl.logger.Errorw("err in creating cdWorkflowRunner, ManualCdTrigger", "cdWorkflowId", cdWorkflowId, "err", err) - return 0, err - } - runner.CdWorkflow = &pipelineConfig.CdWorkflow{ - Pipeline: cdPipeline, - } - overrideRequest.CdWorkflowId = cdWorkflowId - // creating cd pipeline status timeline for deployment initialisation - timeline := impl.pipelineStatusTimelineService.GetTimelineDbObjectByTimelineStatusAndTimelineDescription(savedWfr.Id, 0, pipelineConfig.TIMELINE_STATUS_DEPLOYMENT_INITIATED, pipelineConfig.TIMELINE_DESCRIPTION_DEPLOYMENT_INITIATED, overrideRequest.UserId, time.Now()) - _, span = otel.Tracer("orchestrator").Start(ctx, "cdPipelineStatusTimelineRepo.SaveTimelineForACDHelmApps") - err = impl.pipelineStatusTimelineService.SaveTimeline(timeline, nil, false) - - span.End() - if err != nil { - impl.logger.Errorw("error in creating timeline status for deployment initiation, ManualCdTrigger", "err", err, "timeline", timeline) - } - - //checking vulnerability for deploying image - _, span = otel.Tracer("orchestrator").Start(ctx, "ciArtifactRepository.Get") - artifact, err := impl.ciArtifactRepository.Get(overrideRequest.CiArtifactId) - span.End() - if err != nil { - impl.logger.Errorw("error in getting ciArtifact, ManualCdTrigger", "CiArtifactId", overrideRequest.CiArtifactId, "err", err) - return 0, err - } - // Migration of deprecated DataSource Type - if artifact.IsMigrationRequired() { - migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(artifact.Id) - if migrationErr != nil { - impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", artifact.Id) - } - } - isVulnerable, err := impl.GetArtifactVulnerabilityStatus(artifact, cdPipeline, ctx) - if err != nil { - impl.logger.Errorw("error in getting Artifact vulnerability status, ManualCdTrigger", "err", err) - return 0, err - } - - if isVulnerable == true { - // if image vulnerable, update timeline status and return - if err = impl.MarkCurrentDeploymentFailed(runner, errors.New(pipelineConfig.FOUND_VULNERABILITY), overrideRequest.UserId); err != nil { - impl.logger.Errorw("error while updating current runner status to failed, TriggerDeployment", "wfrId", runner.Id, "err", err) - } - return 0, fmt.Errorf("found vulnerability for image digest %s", artifact.ImageDigest) - } - - // Deploy the release - _, span = otel.Tracer("orchestrator").Start(ctx, "appService.TriggerRelease") - var releaseErr error - releaseId, _, releaseErr = impl.HandleCDTriggerRelease(overrideRequest, ctx, triggeredAt, overrideRequest.UserId) - span.End() - // if releaseErr found, then the mark current deployment Failed and return - if releaseErr != nil { - err := impl.MarkCurrentDeploymentFailed(runner, releaseErr, overrideRequest.UserId) - if err != nil { - impl.logger.Errorw("error while updating current runner status to failed, updatePreviousDeploymentStatus", "cdWfr", runner.Id, "err", err) - } - return 0, releaseErr - } - - // skip updatePreviousDeploymentStatus if Async Install is enabled; handled inside SubscribeDevtronAsyncHelmInstallRequest - if !impl.cdTriggerService.IsDevtronAsyncInstallModeEnabled(cdPipeline.DeploymentAppType) { - // Update previous deployment runner status (in transaction): Failed - _, span = otel.Tracer("orchestrator").Start(ctx, "updatePreviousDeploymentStatus") - err1 := impl.updatePreviousDeploymentStatus(runner, cdPipeline.Id, triggeredAt, overrideRequest.UserId) - span.End() - if err1 != nil { - impl.logger.Errorw("error while update previous cd workflow runners, ManualCdTrigger", "err", err, "runner", runner, "pipelineId", cdPipeline.Id) - return 0, err1 - } - } - - if overrideRequest.DeploymentAppType == util.PIPELINE_DEPLOYMENT_TYPE_MANIFEST_DOWNLOAD { - runner := &pipelineConfig.CdWorkflowRunner{ - Id: runner.Id, - Name: cdPipeline.Name, - WorkflowType: bean.CD_WORKFLOW_TYPE_DEPLOY, - ExecutorType: pipelineConfig.WORKFLOW_EXECUTOR_TYPE_AWF, - TriggeredBy: overrideRequest.UserId, - StartedOn: triggeredAt, - Status: pipelineConfig.WorkflowSucceeded, - Namespace: impl.config.GetDefaultNamespace(), - CdWorkflowId: overrideRequest.CdWorkflowId, - AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: overrideRequest.UserId, UpdatedOn: triggeredAt, UpdatedBy: overrideRequest.UserId}, - } - updateErr := impl.cdWorkflowRepository.UpdateWorkFlowRunner(runner) - if updateErr != nil { - impl.logger.Errorw("error in updating runner for manifest_download type, ManualCdTrigger", "CdWorkflowId", overrideRequest.CdWorkflowId, "err", err) - return 0, updateErr - } - } - - case bean.CD_WORKFLOW_TYPE_POST: - cdWfRunner, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(ctx, overrideRequest.CdWorkflowId, bean.CD_WORKFLOW_TYPE_DEPLOY) - if err != nil && !util.IsErrNoRows(err) { - impl.logger.Errorw("err in getting cdWorkflowRunner, ManualCdTrigger", "cdWorkflowId", overrideRequest.CdWorkflowId, "err", err) - return 0, err - } - - var cdWf *pipelineConfig.CdWorkflow - if cdWfRunner.CdWorkflowId == 0 { - cdWf = &pipelineConfig.CdWorkflow{ - CiArtifactId: overrideRequest.CiArtifactId, - PipelineId: overrideRequest.PipelineId, - AuditLog: sql.AuditLog{CreatedOn: triggeredAt, CreatedBy: overrideRequest.UserId, UpdatedOn: triggeredAt, UpdatedBy: overrideRequest.UserId}, - } - err := impl.cdWorkflowRepository.SaveWorkFlow(ctx, cdWf) - if err != nil { - impl.logger.Errorw("error in creating cdWorkflow, ManualCdTrigger", "CdWorkflowId", overrideRequest.CdWorkflowId, "err", err) - return 0, err - } - } else { - _, span = otel.Tracer("orchestrator").Start(ctx, "cdWorkflowRepository.FindById") - cdWf, err = impl.cdWorkflowRepository.FindById(overrideRequest.CdWorkflowId) - span.End() - if err != nil && !util.IsErrNoRows(err) { - impl.logger.Errorw("error in getting cdWorkflow, ManualCdTrigger", "CdWorkflowId", overrideRequest.CdWorkflowId, "err", err) - return 0, err - } - } - _, span = otel.Tracer("orchestrator").Start(ctx, "TriggerPostStage") - triggerRequest := TriggerRequest{ - CdWf: cdWf, - Pipeline: cdPipeline, - TriggeredBy: overrideRequest.UserId, - RefCdWorkflowRunnerId: 0, - TriggerContext: triggerContext, - } - err = impl.TriggerPostStage(triggerRequest) - span.End() - if err != nil { - impl.logger.Errorw("error in TriggerPostStage, ManualCdTrigger", "CdWorkflowId", cdWf.Id, "err", err) - return 0, err - } - default: - impl.logger.Errorw("invalid CdWorkflowType, ManualCdTrigger", "CdWorkflowType", overrideRequest.CdWorkflowType, "err", err) - return 0, fmt.Errorf("invalid CdWorkflowType %s for the trigger request", string(overrideRequest.CdWorkflowType)) - } - - return releaseId, err -} - -type DeploymentGroupAppWithEnv struct { - EnvironmentId int `json:"environmentId"` - DeploymentGroupId int `json:"deploymentGroupId"` - AppId int `json:"appId"` - Active bool `json:"active"` - UserId int32 `json:"userId"` - RequestType RequestType `json:"requestType" validate:"oneof=START STOP"` -} - -func (impl *WorkflowDagExecutorImpl) TriggerBulkHibernateAsync(request StopDeploymentGroupRequest, ctx context.Context) (interface{}, error) { - dg, err := impl.groupRepository.FindByIdWithApp(request.DeploymentGroupId) - if err != nil { - impl.logger.Errorw("error while fetching dg", "err", err) - return nil, err - } - - for _, app := range dg.DeploymentGroupApps { - deploymentGroupAppWithEnv := &DeploymentGroupAppWithEnv{ - AppId: app.AppId, - EnvironmentId: dg.EnvironmentId, - DeploymentGroupId: dg.Id, - Active: dg.Active, - UserId: request.UserId, - RequestType: request.RequestType, - } - - data, err := json.Marshal(deploymentGroupAppWithEnv) - if err != nil { - impl.logger.Errorw("error while writing app stop event to nats ", "app", app.AppId, "deploymentGroup", app.DeploymentGroupId, "err", err) - } else { - err = impl.pubsubClient.Publish(pubsub.BULK_HIBERNATE_TOPIC, string(data)) - if err != nil { - impl.logger.Errorw("Error while publishing request", "topic", pubsub.BULK_HIBERNATE_TOPIC, "error", err) - } - } - } - return nil, nil -} - -func extractTimelineFailedStatusDetails(err error) string { - errorString := util.GetGRPCErrorDetailedMessage(err) - switch errorString { - case pipelineConfig.FOUND_VULNERABILITY: - return pipelineConfig.TIMELINE_DESCRIPTION_VULNERABLE_IMAGE - default: - return util.GetTruncatedMessage(fmt.Sprintf("Deployment failed: %s", errorString), 255) - } -} - -func (impl *WorkflowDagExecutorImpl) MarkCurrentDeploymentFailed(runner *pipelineConfig.CdWorkflowRunner, releaseErr error, triggeredBy int32) error { - err := impl.pipelineStatusTimelineService.MarkPipelineStatusTimelineFailed(runner.Id, extractTimelineFailedStatusDetails(releaseErr)) - if err != nil { - impl.logger.Errorw("error updating CdPipelineStatusTimeline", "err", err, "releaseErr", releaseErr) - return err - } - //update current WF with error status - impl.logger.Errorw("error in triggering cd WF, setting wf status as fail ", "wfId", runner.Id, "err", releaseErr) - runner.Status = pipelineConfig.WorkflowFailed - runner.Message = util.GetGRPCErrorDetailedMessage(releaseErr) - runner.FinishedOn = time.Now() - runner.UpdatedOn = time.Now() - runner.UpdatedBy = triggeredBy - err1 := impl.cdWorkflowRepository.UpdateWorkFlowRunner(runner) - if err1 != nil { - impl.logger.Errorw("error updating cd wf runner status", "err", releaseErr, "currentRunner", runner) - return err1 - } - util4.TriggerCDMetrics(pipelineConfig.GetTriggerMetricsFromRunnerObj(runner), impl.config.ExposeCDMetrics) - return nil -} - -func (impl *WorkflowDagExecutorImpl) HandleCDTriggerRelease(overrideRequest *bean.ValuesOverrideRequest, ctx context.Context, triggeredAt time.Time, deployedBy int32) (releaseNo int, manifest []byte, err error) { - if impl.cdTriggerService.IsDevtronAsyncInstallModeEnabled(overrideRequest.DeploymentAppType) { - // asynchronous mode of installation starts - return impl.workflowEventPublishService.TriggerHelmAsyncRelease(overrideRequest, ctx, triggeredAt, deployedBy) - } - // synchronous mode of installation starts - - valuesOverrideResponse, builtChartPath, err := impl.manifestCreationService.BuildManifestForTrigger(overrideRequest, triggeredAt, ctx) - _, span := otel.Tracer("orchestrator").Start(ctx, "CreateHistoriesForDeploymentTrigger") - err1 := impl.deployedConfigurationHistoryService.CreateHistoriesForDeploymentTrigger(valuesOverrideResponse.Pipeline, valuesOverrideResponse.PipelineStrategy, valuesOverrideResponse.EnvOverride, triggeredAt, deployedBy) - if err1 != nil { - impl.logger.Errorw("error in saving histories for trigger", "err", err1, "pipelineId", valuesOverrideResponse.Pipeline.Id, "wfrId", overrideRequest.WfrId) - } - span.End() - if err != nil { - impl.logger.Errorw("error in building merged manifest for trigger", "err", err) - return releaseNo, manifest, err - } - return impl.cdTriggerService.TriggerRelease(overrideRequest, valuesOverrideResponse, builtChartPath, ctx, triggeredAt, deployedBy) -} - -func (impl *WorkflowDagExecutorImpl) TriggerCD(artifact *repository.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, triggeredAt time.Time) error { - impl.logger.Debugw("automatic pipeline trigger attempt async", "artifactId", artifact.Id) - - return impl.triggerReleaseAsync(artifact, cdWorkflowId, wfrId, pipeline, triggeredAt) -} - -func (impl *WorkflowDagExecutorImpl) triggerReleaseAsync(artifact *repository.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, triggeredAt time.Time) error { - err := impl.validateAndTrigger(pipeline, artifact, cdWorkflowId, wfrId, triggeredAt) - if err != nil { - impl.logger.Errorw("error in trigger for pipeline", "pipelineId", strconv.Itoa(pipeline.Id)) - } - impl.logger.Debugw("trigger attempted for all pipeline ", "artifactId", artifact.Id) - return err -} - -func (impl *WorkflowDagExecutorImpl) validateAndTrigger(p *pipelineConfig.Pipeline, artifact *repository.CiArtifact, cdWorkflowId, wfrId int, triggeredAt time.Time) error { - object := impl.enforcerUtil.GetAppRBACNameByAppId(p.AppId) - envApp := strings.Split(object, "/") - if len(envApp) != 2 { - impl.logger.Error("invalid req, app and env not found from rbac") - return errors.New("invalid req, app and env not found from rbac") - } - err := impl.releasePipeline(p, artifact, cdWorkflowId, wfrId, triggeredAt) - return err -} - -func (impl *WorkflowDagExecutorImpl) releasePipeline(pipeline *pipelineConfig.Pipeline, artifact *repository.CiArtifact, cdWorkflowId, wfrId int, triggeredAt time.Time) error { - impl.logger.Debugw("triggering release for ", "cdPipelineId", pipeline.Id, "artifactId", artifact.Id) - - pipeline, err := impl.pipelineRepository.FindById(pipeline.Id) - if err != nil { - impl.logger.Errorw("error in fetching pipeline by pipelineId", "err", err) - return err - } - - request := &bean.ValuesOverrideRequest{ - PipelineId: pipeline.Id, - UserId: artifact.CreatedBy, - CiArtifactId: artifact.Id, - AppId: pipeline.AppId, - CdWorkflowId: cdWorkflowId, - ForceTrigger: true, - DeploymentWithConfig: bean.DEPLOYMENT_CONFIG_TYPE_LAST_SAVED, - WfrId: wfrId, - } - impl.SetPipelineFieldsInOverrideRequest(request, pipeline) - - ctx, err := impl.argoUserService.BuildACDContext() - if err != nil { - impl.logger.Errorw("error in creating acd sync context", "pipelineId", pipeline.Id, "artifactId", artifact.Id, "err", err) - return err - } - //setting deployedBy as 1(system user) since case of auto trigger - id, _, err := impl.HandleCDTriggerRelease(request, ctx, triggeredAt, 1) - if err != nil { - impl.logger.Errorw("error in auto cd pipeline trigger", "pipelineId", pipeline.Id, "artifactId", artifact.Id, "err", err) - } else { - impl.logger.Infow("pipeline successfully triggered ", "cdPipelineId", pipeline.Id, "artifactId", artifact.Id, "releaseId", id) - } - return err - -} - -func (impl *WorkflowDagExecutorImpl) SetPipelineFieldsInOverrideRequest(overrideRequest *bean.ValuesOverrideRequest, pipeline *pipelineConfig.Pipeline) { - overrideRequest.PipelineId = pipeline.Id - overrideRequest.PipelineName = pipeline.Name - overrideRequest.EnvId = pipeline.EnvironmentId - overrideRequest.EnvName = pipeline.Environment.Name - overrideRequest.ClusterId = pipeline.Environment.ClusterId - overrideRequest.AppId = pipeline.AppId - overrideRequest.AppName = pipeline.App.AppName - overrideRequest.DeploymentAppType = pipeline.DeploymentAppType -} - -// write integration/unit test for each function - -func (impl *WorkflowDagExecutorImpl) updatePipeline(pipeline *pipelineConfig.Pipeline, userId int32) (bool, error) { - err := impl.pipelineRepository.SetDeploymentAppCreatedInPipeline(true, pipeline.Id, userId) - if err != nil { - impl.logger.Errorw("error on updating cd pipeline for setting deployment app created", "err", err) - return false, err - } - return true, nil -} - -// canInitiateTrigger checks if the current trigger request with natsMsgId haven't already initiated the trigger. -// throws error if the request is already processed. -func (impl *WorkflowDagExecutorImpl) canInitiateTrigger(natsMsgId string) (bool, error) { - if natsMsgId == "" { - return true, nil - } - exists, err := impl.cdWorkflowRepository.CheckWorkflowRunnerByReferenceId(natsMsgId) - if err != nil { - impl.logger.Errorw("error in fetching cd workflow runner using reference_id", "referenceId", natsMsgId, "err", err) - return false, errors.New("error in fetching cd workflow runner") - } - - if exists { - impl.logger.Errorw("duplicate pre stage trigger request as there is already a workflow runner object created by this message") - return false, errors.New("duplicate pre stage trigger request, this request was already processed") - } - - return true, nil -} - -// GetTriggerValidateFuncs gets all the required validation funcs -func (impl *WorkflowDagExecutorImpl) GetTriggerValidateFuncs() []pubsub.ValidateMsg { - - var duplicateTriggerValidateFunc pubsub.ValidateMsg = func(msg model.PubSubMsg) bool { - if msg.MsgDeliverCount == 1 { - // first time message got delivered, always validate this. - return true - } - - // message is redelivered, check if the message is already processed. - if ok, err := impl.canInitiateTrigger(msg.MsgId); !ok || err != nil { - impl.logger.Warnw("duplicate trigger condition, duplicate message", "msgId", msg.MsgId, "err", err) - return false - } - return true - } - - return []pubsub.ValidateMsg{duplicateTriggerValidateFunc} - -} diff --git a/pkg/workflow/cd/CdWorkflowCommonService.go b/pkg/workflow/cd/CdWorkflowCommonService.go index c53178f105..0fda3767f0 100644 --- a/pkg/workflow/cd/CdWorkflowCommonService.go +++ b/pkg/workflow/cd/CdWorkflowCommonService.go @@ -2,9 +2,14 @@ package cd import ( "context" + "errors" "fmt" + pubsub "github.com/devtron-labs/common-lib/pubsub-lib" + "github.com/devtron-labs/common-lib/pubsub-lib/model" + "github.com/devtron-labs/common-lib/utils/k8s/health" "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app/status" "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/sql" @@ -16,7 +21,11 @@ import ( ) type CdWorkflowCommonService interface { + UpdatePreviousDeploymentStatus(currentRunner *pipelineConfig.CdWorkflowRunner, pipelineId int, triggeredAt time.Time, triggeredBy int32) error + MarkCurrentDeploymentFailed(runner *pipelineConfig.CdWorkflowRunner, releaseErr error, triggeredBy int32) error UpdateCDWorkflowRunnerStatus(ctx context.Context, overrideRequest *bean.ValuesOverrideRequest, triggeredAt time.Time, status, message string) error + + GetTriggerValidateFuncs() []pubsub.ValidateMsg } type CdWorkflowCommonServiceImpl struct { @@ -25,27 +34,125 @@ type CdWorkflowCommonServiceImpl struct { pipelineStatusTimelineService status.PipelineStatusTimelineService //TODO: remove below - config *types.CdConfig - pipelineRepository pipelineConfig.PipelineRepository + config *types.CdConfig + pipelineRepository pipelineConfig.PipelineRepository + pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository } func NewCdWorkflowCommonServiceImpl(logger *zap.SugaredLogger, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, pipelineStatusTimelineService status.PipelineStatusTimelineService, - pipelineRepository pipelineConfig.PipelineRepository) (*CdWorkflowCommonServiceImpl, error) { + pipelineRepository pipelineConfig.PipelineRepository, + pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository) (*CdWorkflowCommonServiceImpl, error) { config, err := types.GetCdConfig() if err != nil { return nil, err } return &CdWorkflowCommonServiceImpl{ - logger: logger, - cdWorkflowRepository: cdWorkflowRepository, - pipelineStatusTimelineService: pipelineStatusTimelineService, - config: config, - pipelineRepository: pipelineRepository, + logger: logger, + cdWorkflowRepository: cdWorkflowRepository, + pipelineStatusTimelineService: pipelineStatusTimelineService, + config: config, + pipelineRepository: pipelineRepository, + pipelineStatusTimelineRepository: pipelineStatusTimelineRepository, }, nil } +func (impl *CdWorkflowCommonServiceImpl) UpdatePreviousDeploymentStatus(currentRunner *pipelineConfig.CdWorkflowRunner, pipelineId int, triggeredAt time.Time, triggeredBy int32) error { + // Initiating DB transaction + dbConnection := impl.cdWorkflowRepository.GetConnection() + tx, err := dbConnection.Begin() + if err != nil { + impl.logger.Errorw("error on update status, txn begin failed", "err", err) + return err + } + // Rollback tx on error. + defer tx.Rollback() + + //update [n,n-1] statuses as failed if not terminal + terminalStatus := []string{string(health.HealthStatusHealthy), pipelineConfig.WorkflowAborted, pipelineConfig.WorkflowFailed, pipelineConfig.WorkflowSucceeded} + previousNonTerminalRunners, err := impl.cdWorkflowRepository.FindPreviousCdWfRunnerByStatus(pipelineId, currentRunner.Id, terminalStatus) + if err != nil { + impl.logger.Errorw("error fetching previous wf runner, updating cd wf runner status,", "err", err, "currentRunner", currentRunner) + return err + } else if len(previousNonTerminalRunners) == 0 { + impl.logger.Errorw("no previous runner found in updating cd wf runner status,", "err", err, "currentRunner", currentRunner) + return nil + } + + var timelines []*pipelineConfig.PipelineStatusTimeline + for _, previousRunner := range previousNonTerminalRunners { + if previousRunner.Status == string(health.HealthStatusHealthy) || + previousRunner.Status == pipelineConfig.WorkflowSucceeded || + previousRunner.Status == pipelineConfig.WorkflowAborted || + previousRunner.Status == pipelineConfig.WorkflowFailed { + //terminal status return + impl.logger.Infow("skip updating cd wf runner status as previous runner status is", "status", previousRunner.Status) + continue + } + impl.logger.Infow("updating cd wf runner status as previous runner status is", "status", previousRunner.Status) + previousRunner.FinishedOn = triggeredAt + previousRunner.Message = pipelineConfig.NEW_DEPLOYMENT_INITIATED + previousRunner.Status = pipelineConfig.WorkflowFailed + previousRunner.UpdatedOn = time.Now() + previousRunner.UpdatedBy = triggeredBy + timeline := &pipelineConfig.PipelineStatusTimeline{ + CdWorkflowRunnerId: previousRunner.Id, + Status: pipelineConfig.TIMELINE_STATUS_DEPLOYMENT_SUPERSEDED, + StatusDetail: "This deployment is superseded.", + StatusTime: time.Now(), + AuditLog: sql.AuditLog{ + CreatedBy: 1, + CreatedOn: time.Now(), + UpdatedBy: 1, + UpdatedOn: time.Now(), + }, + } + timelines = append(timelines, timeline) + } + + err = impl.cdWorkflowRepository.UpdateWorkFlowRunners(previousNonTerminalRunners) + if err != nil { + impl.logger.Errorw("error updating cd wf runner status", "err", err, "previousNonTerminalRunners", previousNonTerminalRunners) + return err + } + err = impl.pipelineStatusTimelineRepository.SaveTimelinesWithTxn(timelines, tx) + if err != nil { + impl.logger.Errorw("error updating pipeline status timelines", "err", err, "timelines", timelines) + return err + } + //commit transaction + err = tx.Commit() + if err != nil { + impl.logger.Errorw("error in db transaction commit", "err", err) + return err + } + return nil + +} + +func (impl *CdWorkflowCommonServiceImpl) MarkCurrentDeploymentFailed(runner *pipelineConfig.CdWorkflowRunner, releaseErr error, triggeredBy int32) error { + err := impl.pipelineStatusTimelineService.MarkPipelineStatusTimelineFailed(runner.Id, extractTimelineFailedStatusDetails(releaseErr)) + if err != nil { + impl.logger.Errorw("error updating CdPipelineStatusTimeline", "err", err, "releaseErr", releaseErr) + return err + } + //update current WF with error status + impl.logger.Errorw("error in triggering cd WF, setting wf status as fail ", "wfId", runner.Id, "err", releaseErr) + runner.Status = pipelineConfig.WorkflowFailed + runner.Message = util.GetGRPCErrorDetailedMessage(releaseErr) + runner.FinishedOn = time.Now() + runner.UpdatedOn = time.Now() + runner.UpdatedBy = triggeredBy + err1 := impl.cdWorkflowRepository.UpdateWorkFlowRunner(runner) + if err1 != nil { + impl.logger.Errorw("error updating cd wf runner status", "err", releaseErr, "currentRunner", runner) + return err1 + } + util4.TriggerCDMetrics(pipelineConfig.GetTriggerMetricsFromRunnerObj(runner), impl.config.ExposeCDMetrics) + return nil +} + // TODO: remove bean.ValuesOverrideRequest func (impl *CdWorkflowCommonServiceImpl) UpdateCDWorkflowRunnerStatus(ctx context.Context, overrideRequest *bean.ValuesOverrideRequest, triggeredAt time.Time, status, message string) error { // In case of terminal status update finished on time @@ -130,3 +237,49 @@ func (impl *CdWorkflowCommonServiceImpl) UpdateCDWorkflowRunnerStatus(ctx contex } return nil } + +func extractTimelineFailedStatusDetails(err error) string { + errorString := util.GetGRPCErrorDetailedMessage(err) + switch errorString { + case pipelineConfig.FOUND_VULNERABILITY: + return pipelineConfig.TIMELINE_DESCRIPTION_VULNERABLE_IMAGE + default: + return util.GetTruncatedMessage(fmt.Sprintf("Deployment failed: %s", errorString), 255) + } +} + +// GetTriggerValidateFuncs gets all the required validation funcs +func (impl *CdWorkflowCommonServiceImpl) GetTriggerValidateFuncs() []pubsub.ValidateMsg { + var duplicateTriggerValidateFunc pubsub.ValidateMsg = func(msg model.PubSubMsg) bool { + if msg.MsgDeliverCount == 1 { + // first time message got delivered, always validate this. + return true + } + // message is redelivered, check if the message is already processed. + if ok, err := impl.canInitiateTrigger(msg.MsgId); !ok || err != nil { + impl.logger.Warnw("duplicate trigger condition, duplicate message", "msgId", msg.MsgId, "err", err) + return false + } + return true + } + return []pubsub.ValidateMsg{duplicateTriggerValidateFunc} +} + +// canInitiateTrigger checks if the current trigger request with natsMsgId haven't already initiated the trigger. +// throws error if the request is already processed. +func (impl *CdWorkflowCommonServiceImpl) canInitiateTrigger(natsMsgId string) (bool, error) { + if natsMsgId == "" { + return true, nil + } + exists, err := impl.cdWorkflowRepository.CheckWorkflowRunnerByReferenceId(natsMsgId) + if err != nil { + impl.logger.Errorw("error in fetching cd workflow runner using reference_id", "referenceId", natsMsgId, "err", err) + return false, errors.New("error in fetching cd workflow runner") + } + + if exists { + impl.logger.Errorw("duplicate pre stage trigger request as there is already a workflow runner object created by this message") + return false, errors.New("duplicate pre stage trigger request, this request was already processed") + } + return true, nil +} diff --git a/pkg/workflow/dag/WorkflowDagExecutor.go b/pkg/workflow/dag/WorkflowDagExecutor.go new file mode 100644 index 0000000000..7320828d9b --- /dev/null +++ b/pkg/workflow/dag/WorkflowDagExecutor.go @@ -0,0 +1,1229 @@ +/* + * Copyright (c) 2020 Devtron Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package dag + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" + bean6 "github.com/devtron-labs/devtron/api/helm-app/bean" + client2 "github.com/devtron-labs/devtron/api/helm-app/service" + client "github.com/devtron-labs/devtron/client/events" + "github.com/devtron-labs/devtron/pkg/build/artifacts" + "github.com/devtron-labs/devtron/pkg/deployment/manifest" + "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps" + bean5 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" + bean7 "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" + "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/pipeline/executors" + repository2 "github.com/devtron-labs/devtron/pkg/plugin/repository" + "github.com/devtron-labs/devtron/pkg/sql" + "github.com/devtron-labs/devtron/pkg/workflow/cd" + bean2 "github.com/devtron-labs/devtron/pkg/workflow/dag/bean" + util2 "github.com/devtron-labs/devtron/util/event" + "sync" + "time" + + "github.com/devtron-labs/common-lib/pubsub-lib/model" + "github.com/devtron-labs/devtron/client/argocdServer/application" + bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + repository4 "github.com/devtron-labs/devtron/pkg/pipeline/repository" + "github.com/devtron-labs/devtron/pkg/pipeline/types" + serverBean "github.com/devtron-labs/devtron/pkg/server/bean" + util4 "github.com/devtron-labs/devtron/util" + "github.com/pkg/errors" + "go.opentelemetry.io/otel" + "k8s.io/utils/strings/slices" + + pubsub "github.com/devtron-labs/common-lib/pubsub-lib" + "github.com/devtron-labs/devtron/api/bean" + "github.com/devtron-labs/devtron/internal/sql/models" + "github.com/devtron-labs/devtron/internal/sql/repository" + "github.com/devtron-labs/devtron/internal/sql/repository/appWorkflow" + "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/app" + "github.com/devtron-labs/devtron/util/rbac" + "github.com/go-pg/pg" + "go.uber.org/zap" +) + +type WorkflowDagExecutor interface { + HandleCiSuccessEvent(triggerContext bean5.TriggerContext, ciPipelineId int, request *bean2.CiArtifactWebhookRequest, imagePushedAt *time.Time) (id int, err error) + HandlePreStageSuccessEvent(triggerContext bean5.TriggerContext, cdStageCompleteEvent bean7.CdStageCompleteEvent) error + HandleDeploymentSuccessEvent(triggerContext bean5.TriggerContext, pipelineOverride *chartConfig.PipelineOverride) error + HandlePostStageSuccessEvent(triggerContext bean5.TriggerContext, cdWorkflowId int, cdPipelineId int, triggeredBy int32, pluginRegistryImageDetails map[string][]string) error + HandleCdStageReTrigger(runner *pipelineConfig.CdWorkflowRunner) error + HandleCiStepFailedEvent(ciPipelineId int, request *bean2.CiArtifactWebhookRequest) (err error) + HandleExternalCiWebhook(externalCiId int, request *bean2.CiArtifactWebhookRequest, + auth func(token string, projectObject string, envObject string) bool, token string) (id int, err error) + + UpdateWorkflowRunnerStatusForDeployment(appIdentifier *client2.AppIdentifier, wfr *pipelineConfig.CdWorkflowRunner, skipReleaseNotFound bool) bool + OnDeleteCdPipelineEvent(pipelineId int, triggeredBy int32) + + BuildCiArtifactRequestForWebhook(event pipeline.ExternalCiWebhookDto) (*bean2.CiArtifactWebhookRequest, error) +} + +type WorkflowDagExecutorImpl struct { + logger *zap.SugaredLogger + pipelineRepository pipelineConfig.PipelineRepository + cdWorkflowRepository pipelineConfig.CdWorkflowRepository + pubsubClient *pubsub.PubSubClientServiceImpl + ciArtifactRepository repository.CiArtifactRepository + enforcerUtil rbac.EnforcerUtil + appWorkflowRepository appWorkflow.AppWorkflowRepository + pipelineStageService pipeline.PipelineStageService + ciWorkflowRepository pipelineConfig.CiWorkflowRepository + ciPipelineRepository pipelineConfig.CiPipelineRepository + pipelineStageRepository repository4.PipelineStageRepository + globalPluginRepository repository2.GlobalPluginRepository + config *types.CdConfig + ciConfig *types.CiConfig + appServiceConfig *app.AppServiceConfig + eventClient client.EventClient + eventFactory client.EventFactory + customTagService pipeline.CustomTagService + + devtronAsyncHelmInstallRequestMap map[int]bool + devtronAsyncHelmInstallRequestLock *sync.Mutex + devtronAppReleaseContextMap map[int]DevtronAppReleaseContextType + devtronAppReleaseContextMapLock *sync.Mutex + + helmAppService client2.HelmAppService + + cdWorkflowCommonService cd.CdWorkflowCommonService + cdTriggerService devtronApps.TriggerService + + manifestCreationService manifest.ManifestCreationService + commonArtifactService artifacts.CommonArtifactService +} + +type DevtronAppReleaseContextType struct { + CancelContext context.CancelFunc + RunnerId int +} + +func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pipelineConfig.PipelineRepository, + cdWorkflowRepository pipelineConfig.CdWorkflowRepository, + pubsubClient *pubsub.PubSubClientServiceImpl, + ciArtifactRepository repository.CiArtifactRepository, + enforcerUtil rbac.EnforcerUtil, + appWorkflowRepository appWorkflow.AppWorkflowRepository, + pipelineStageService pipeline.PipelineStageService, + ciWorkflowRepository pipelineConfig.CiWorkflowRepository, + ciPipelineRepository pipelineConfig.CiPipelineRepository, + pipelineStageRepository repository4.PipelineStageRepository, + globalPluginRepository repository2.GlobalPluginRepository, + eventClient client.EventClient, + eventFactory client.EventFactory, + helmAppService client2.HelmAppService, + pipelineConfigListenerService pipeline.PipelineConfigListenerService, + cdWorkflowCommonService cd.CdWorkflowCommonService, + cdTriggerService devtronApps.TriggerService, + manifestCreationService manifest.ManifestCreationService, + commonArtifactService artifacts.CommonArtifactService) *WorkflowDagExecutorImpl { + wde := &WorkflowDagExecutorImpl{logger: Logger, + pipelineRepository: pipelineRepository, + cdWorkflowRepository: cdWorkflowRepository, + pubsubClient: pubsubClient, + ciArtifactRepository: ciArtifactRepository, + enforcerUtil: enforcerUtil, + appWorkflowRepository: appWorkflowRepository, + pipelineStageService: pipelineStageService, + ciWorkflowRepository: ciWorkflowRepository, + ciPipelineRepository: ciPipelineRepository, + pipelineStageRepository: pipelineStageRepository, + globalPluginRepository: globalPluginRepository, + eventClient: eventClient, + eventFactory: eventFactory, + + devtronAsyncHelmInstallRequestMap: make(map[int]bool), + devtronAsyncHelmInstallRequestLock: &sync.Mutex{}, + devtronAppReleaseContextMap: make(map[int]DevtronAppReleaseContextType), + devtronAppReleaseContextMapLock: &sync.Mutex{}, + helmAppService: helmAppService, + cdWorkflowCommonService: cdWorkflowCommonService, + cdTriggerService: cdTriggerService, + manifestCreationService: manifestCreationService, + commonArtifactService: commonArtifactService, + } + config, err := types.GetCdConfig() + if err != nil { + return nil + } + wde.config = config + ciConfig, err := types.GetCiConfig() + if err != nil { + return nil + } + wde.ciConfig = ciConfig + appServiceConfig, err := app.GetAppServiceConfig() + if err != nil { + return nil + } + wde.appServiceConfig = appServiceConfig + err = wde.SubscribeDevtronAsyncHelmInstallRequest() + if err != nil { + return nil + } + pipelineConfigListenerService.RegisterPipelineDeleteListener(wde) + return wde +} + +func (impl *WorkflowDagExecutorImpl) HandleCdStageReTrigger(runner *pipelineConfig.CdWorkflowRunner) error { + // do not re-trigger if retries = 0 + if !impl.config.WorkflowRetriesEnabled() { + impl.logger.Debugw("cd stage workflow re-triggering is not enabled") + return nil + } + + impl.logger.Infow("re triggering cd stage ", "runnerId", runner.Id) + var err error + // add comment for this logic + if runner.RefCdWorkflowRunnerId != 0 { + runner, err = impl.cdWorkflowRepository.FindWorkflowRunnerById(runner.RefCdWorkflowRunnerId) + if err != nil { + impl.logger.Errorw("error in FindWorkflowRunnerById by id ", "err", err, "wfrId", runner.RefCdWorkflowRunnerId) + return err + } + } + retryCnt, err := impl.cdWorkflowRepository.FindRetriedWorkflowCountByReferenceId(runner.Id) + if err != nil { + impl.logger.Errorw("error in FindRetriedWorkflowCountByReferenceId ", "err", err, "cdWorkflowRunnerId", runner.Id) + return err + } + + if retryCnt >= impl.config.MaxCdWorkflowRunnerRetries { + impl.logger.Infow("maximum retries for this workflow are exhausted, not re-triggering again", "retries", retryCnt, "wfrId", runner.Id) + return nil + } + + triggerRequest := bean5.TriggerRequest{ + CdWf: runner.CdWorkflow, + Pipeline: runner.CdWorkflow.Pipeline, + Artifact: runner.CdWorkflow.CiArtifact, + TriggeredBy: 1, + ApplyAuth: false, + RefCdWorkflowRunnerId: runner.Id, + TriggerContext: bean5.TriggerContext{ + Context: context.Background(), + }, + } + + if runner.WorkflowType == bean.CD_WORKFLOW_TYPE_PRE { + err = impl.cdTriggerService.TriggerPreStage(triggerRequest) + if err != nil { + impl.logger.Errorw("error in TriggerPreStage ", "err", err, "cdWorkflowRunnerId", runner.Id) + return err + } + } else if runner.WorkflowType == bean.CD_WORKFLOW_TYPE_POST { + err = impl.cdTriggerService.TriggerPostStage(triggerRequest) + if err != nil { + impl.logger.Errorw("error in TriggerPostStage ", "err", err, "cdWorkflowRunnerId", runner.Id) + return err + } + } + + impl.logger.Infow("cd stage re triggered for runner", "runnerId", runner.Id) + return nil +} + +func (impl *WorkflowDagExecutorImpl) extractOverrideRequestFromCDAsyncInstallEvent(msg *model.PubSubMsg) (*bean7.AsyncCdDeployEvent, *client2.AppIdentifier, error) { + CDAsyncInstallNatsMessage := &bean7.AsyncCdDeployEvent{} + err := json.Unmarshal([]byte(msg.Data), CDAsyncInstallNatsMessage) + if err != nil { + impl.logger.Errorw("error in unmarshalling CD async install request nats message", "err", err) + return nil, nil, err + } + pipeline, err := impl.pipelineRepository.FindById(CDAsyncInstallNatsMessage.ValuesOverrideRequest.PipelineId) + if err != nil { + impl.logger.Errorw("error in fetching pipeline by pipelineId", "err", err) + return nil, nil, err + } + devtronApps.SetPipelineFieldsInOverrideRequest(CDAsyncInstallNatsMessage.ValuesOverrideRequest, pipeline) + if CDAsyncInstallNatsMessage.ValuesOverrideRequest.DeploymentType == models.DEPLOYMENTTYPE_UNKNOWN { + CDAsyncInstallNatsMessage.ValuesOverrideRequest.DeploymentType = models.DEPLOYMENTTYPE_DEPLOY + } + appIdentifier := &client2.AppIdentifier{ + ClusterId: pipeline.Environment.ClusterId, + Namespace: pipeline.Environment.Namespace, + ReleaseName: pipeline.DeploymentAppName, + } + return CDAsyncInstallNatsMessage, appIdentifier, nil +} + +// UpdateWorkflowRunnerStatusForDeployment will update CD workflow runner based on release status and app status +func (impl *WorkflowDagExecutorImpl) UpdateWorkflowRunnerStatusForDeployment(appIdentifier *client2.AppIdentifier, wfr *pipelineConfig.CdWorkflowRunner, skipReleaseNotFound bool) bool { + helmInstalledDevtronApp, err := impl.helmAppService.GetApplicationAndReleaseStatus(context.Background(), appIdentifier) + if err != nil { + impl.logger.Errorw("error in getting helm app release status", "appIdentifier", appIdentifier, "err", err) + // Handle release not found errors + if skipReleaseNotFound && util.GetGRPCErrorDetailedMessage(err) != bean6.ErrReleaseNotFound { + // skip this error and continue for next workflow status + impl.logger.Warnw("found error, skipping helm apps status update for this trigger", "appIdentifier", appIdentifier, "err", err) + return false + } + // If release not found, mark the deployment as failure + wfr.Status = pipelineConfig.WorkflowFailed + wfr.Message = util.GetGRPCErrorDetailedMessage(err) + wfr.FinishedOn = time.Now() + return true + } + + switch helmInstalledDevtronApp.GetReleaseStatus() { + case serverBean.HelmReleaseStatusSuperseded: + // If release status is superseded, mark the deployment as failure + wfr.Status = pipelineConfig.WorkflowFailed + wfr.Message = pipelineConfig.NEW_DEPLOYMENT_INITIATED + wfr.FinishedOn = time.Now() + return true + case serverBean.HelmReleaseStatusFailed: + // If release status is failed, mark the deployment as failure + wfr.Status = pipelineConfig.WorkflowFailed + wfr.Message = helmInstalledDevtronApp.GetDescription() + wfr.FinishedOn = time.Now() + return true + case serverBean.HelmReleaseStatusDeployed: + //skip if there is no deployment after wfr.StartedOn and continue for next workflow status + if helmInstalledDevtronApp.GetLastDeployed().AsTime().Before(wfr.StartedOn) { + impl.logger.Warnw("release mismatched, skipping helm apps status update for this trigger", "appIdentifier", appIdentifier, "err", err) + return false + } + + if helmInstalledDevtronApp.GetApplicationStatus() == application.Healthy { + // mark the deployment as succeed + wfr.Status = pipelineConfig.WorkflowSucceeded + wfr.FinishedOn = time.Now() + return true + } + } + if wfr.Status == pipelineConfig.WorkflowInProgress { + return false + } + wfr.Status = pipelineConfig.WorkflowInProgress + return true +} + +func (impl *WorkflowDagExecutorImpl) handleAsyncTriggerReleaseError(releaseErr error, cdWfr *pipelineConfig.CdWorkflowRunner, overrideRequest *bean.ValuesOverrideRequest, appIdentifier *client2.AppIdentifier) { + releaseErrString := util.GetGRPCErrorDetailedMessage(releaseErr) + switch releaseErrString { + case context.DeadlineExceeded.Error(): + // if context deadline is exceeded fetch release status and UpdateWorkflowRunnerStatusForDeployment + if isWfrUpdated := impl.UpdateWorkflowRunnerStatusForDeployment(appIdentifier, cdWfr, false); !isWfrUpdated { + // updating cdWfr to failed + if err := impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(cdWfr, fmt.Errorf("Deployment timeout: release %s took more than %d mins", appIdentifier.ReleaseName, impl.appServiceConfig.DevtronChartInstallRequestTimeout), overrideRequest.UserId); err != nil { + impl.logger.Errorw("error while updating current runner status to failed, handleAsyncTriggerReleaseError", "cdWfr", cdWfr.Id, "err", err) + } + } + cdWfr.UpdatedBy = 1 + cdWfr.UpdatedOn = time.Now() + err := impl.cdWorkflowRepository.UpdateWorkFlowRunner(cdWfr) + if err != nil { + impl.logger.Errorw("error on update cd workflow runner", "wfr", cdWfr, "err", err) + return + } + cdMetrics := util4.CDMetrics{ + AppName: cdWfr.CdWorkflow.Pipeline.DeploymentAppName, + Status: cdWfr.Status, + DeploymentType: cdWfr.CdWorkflow.Pipeline.DeploymentAppType, + EnvironmentName: cdWfr.CdWorkflow.Pipeline.Environment.Name, + Time: time.Since(cdWfr.StartedOn).Seconds() - time.Since(cdWfr.FinishedOn).Seconds(), + } + util4.TriggerCDMetrics(cdMetrics, impl.config.ExposeCDMetrics) + impl.logger.Infow("updated workflow runner status for helm app", "wfr", cdWfr) + return + case context.Canceled.Error(): + if err := impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(cdWfr, errors.New(pipelineConfig.NEW_DEPLOYMENT_INITIATED), overrideRequest.UserId); err != nil { + impl.logger.Errorw("error while updating current runner status to failed, handleAsyncTriggerReleaseError", "cdWfr", cdWfr.Id, "err", err) + } + return + case "": + return + default: + if err := impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(cdWfr, releaseErr, overrideRequest.UserId); err != nil { + impl.logger.Errorw("error while updating current runner status to failed, handleAsyncTriggerReleaseError", "cdWfr", cdWfr.Id, "err", err) + } + return + } +} + +func (impl *WorkflowDagExecutorImpl) handleIfPreviousRunnerTriggerRequest(currentRunner *pipelineConfig.CdWorkflowRunner, userId int32) (bool, error) { + exists, err := impl.cdWorkflowRepository.IsLatestCDWfr(currentRunner.Id, currentRunner.CdWorkflow.PipelineId) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("err on fetching latest cd workflow runner, SubscribeDevtronAsyncHelmInstallRequest", "err", err) + return false, err + } + return exists, nil +} + +func (impl *WorkflowDagExecutorImpl) UpdateReleaseContextForPipeline(pipelineId, cdWfrId int, cancel context.CancelFunc) { + impl.devtronAppReleaseContextMapLock.Lock() + defer impl.devtronAppReleaseContextMapLock.Unlock() + if releaseContext, ok := impl.devtronAppReleaseContextMap[pipelineId]; ok { + //Abort previous running release + impl.logger.Infow("new deployment has been triggered with a running deployment in progress!", "aborting deployment for pipelineId", pipelineId) + releaseContext.CancelContext() + } + impl.devtronAppReleaseContextMap[pipelineId] = DevtronAppReleaseContextType{ + CancelContext: cancel, + RunnerId: cdWfrId, + } +} + +func (impl *WorkflowDagExecutorImpl) RemoveReleaseContextForPipeline(pipelineId int, triggeredBy int32) { + impl.devtronAppReleaseContextMapLock.Lock() + defer impl.devtronAppReleaseContextMapLock.Unlock() + if releaseContext, ok := impl.devtronAppReleaseContextMap[pipelineId]; ok { + //Abort previous running release + impl.logger.Infow("CD pipeline has been deleted with a running deployment in progress!", "aborting deployment for pipelineId", pipelineId) + cdWfr, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(releaseContext.RunnerId) + if err != nil { + impl.logger.Errorw("err on fetching cd workflow runner, RemoveReleaseContextForPipeline", "err", err) + } + if err = impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(cdWfr, errors.New("CD pipeline has been deleted"), triggeredBy); err != nil { + impl.logger.Errorw("error while updating current runner status to failed, RemoveReleaseContextForPipeline", "cdWfr", cdWfr.Id, "err", err) + } + releaseContext.CancelContext() + delete(impl.devtronAppReleaseContextMap, pipelineId) + } + return +} + +func (impl *WorkflowDagExecutorImpl) OnDeleteCdPipelineEvent(pipelineId int, triggeredBy int32) { + impl.logger.Debugw("CD pipeline delete event received", "pipelineId", pipelineId, "deletedBy", triggeredBy) + impl.RemoveReleaseContextForPipeline(pipelineId, triggeredBy) + return +} + +func (impl *WorkflowDagExecutorImpl) isReleaseContextExistsForPipeline(pipelineId, cdWfrId int) bool { + impl.devtronAppReleaseContextMapLock.Lock() + defer impl.devtronAppReleaseContextMapLock.Unlock() + if releaseContext, ok := impl.devtronAppReleaseContextMap[pipelineId]; ok { + return releaseContext.RunnerId == cdWfrId + } + return false +} + +func (impl *WorkflowDagExecutorImpl) handleConcurrentRequest(wfrId int) bool { + impl.devtronAsyncHelmInstallRequestLock.Lock() + defer impl.devtronAsyncHelmInstallRequestLock.Unlock() + if _, exists := impl.devtronAsyncHelmInstallRequestMap[wfrId]; exists { + //request is in process already, Skip here + return true + } + impl.devtronAsyncHelmInstallRequestMap[wfrId] = true + return false +} + +func (impl *WorkflowDagExecutorImpl) cleanUpDevtronAppReleaseContextMap(pipelineId, wfrId int) { + if impl.isReleaseContextExistsForPipeline(pipelineId, wfrId) { + impl.devtronAppReleaseContextMapLock.Lock() + defer impl.devtronAppReleaseContextMapLock.Unlock() + if _, ok := impl.devtronAppReleaseContextMap[pipelineId]; ok { + delete(impl.devtronAppReleaseContextMap, pipelineId) + } + } +} + +func (impl *WorkflowDagExecutorImpl) cleanUpDevtronAsyncHelmInstallRequest(pipelineId, wfrId int) { + impl.devtronAsyncHelmInstallRequestLock.Lock() + defer impl.devtronAsyncHelmInstallRequestLock.Unlock() + if _, exists := impl.devtronAsyncHelmInstallRequestMap[wfrId]; exists { + //request is in process already, Skip here + delete(impl.devtronAsyncHelmInstallRequestMap, wfrId) + } + impl.cleanUpDevtronAppReleaseContextMap(pipelineId, wfrId) +} + +func (impl *WorkflowDagExecutorImpl) processDevtronAsyncHelmInstallRequest(CDAsyncInstallNatsMessage *bean7.AsyncCdDeployEvent, appIdentifier *client2.AppIdentifier) { + overrideRequest := CDAsyncInstallNatsMessage.ValuesOverrideRequest + cdWfr, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(overrideRequest.WfrId) + if err != nil { + impl.logger.Errorw("err on fetching cd workflow runner, processDevtronAsyncHelmInstallRequest", "err", err) + return + } + + // skip if the cdWfr.Status is already in a terminal state + skipCDWfrStatusList := pipelineConfig.WfrTerminalStatusList + skipCDWfrStatusList = append(skipCDWfrStatusList, pipelineConfig.WorkflowInProgress) + if slices.Contains(skipCDWfrStatusList, cdWfr.Status) { + impl.logger.Warnw("skipped deployment as the workflow runner status is already in terminal state, processDevtronAsyncHelmInstallRequest", "cdWfrId", cdWfr.Id, "status", cdWfr.Status) + return + } + + //skip if the cdWfr is not the latest one + exists, err := impl.handleIfPreviousRunnerTriggerRequest(cdWfr, overrideRequest.UserId) + if err != nil { + impl.logger.Errorw("err in validating latest cd workflow runner, processDevtronAsyncHelmInstallRequest", "err", err) + return + } + if exists { + impl.logger.Warnw("skipped deployment as the workflow runner is not the latest one", "cdWfrId", cdWfr.Id) + err := impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(cdWfr, errors.New(pipelineConfig.NEW_DEPLOYMENT_INITIATED), overrideRequest.UserId) + if err != nil { + impl.logger.Errorw("error while updating current runner status to failed, processDevtronAsyncHelmInstallRequest", "cdWfr", cdWfr.Id, "err", err) + return + } + return + } + + if cdWfr.Status == pipelineConfig.WorkflowStarting && impl.isReleaseContextExistsForPipeline(overrideRequest.PipelineId, cdWfr.Id) { + impl.logger.Warnw("event redelivered! deployment is currently in progress, processDevtronAsyncHelmInstallRequest", "cdWfrId", cdWfr.Id, "status", cdWfr.Status) + return + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(impl.appServiceConfig.DevtronChartInstallRequestTimeout)*time.Minute) + defer cancel() + + impl.UpdateReleaseContextForPipeline(overrideRequest.PipelineId, cdWfr.Id, cancel) + //update workflow runner status, used in app workflow view + err = impl.cdWorkflowCommonService.UpdateCDWorkflowRunnerStatus(ctx, overrideRequest, CDAsyncInstallNatsMessage.TriggeredAt, pipelineConfig.WorkflowStarting, "") + if err != nil { + impl.logger.Errorw("error in updating the workflow runner status, processDevtronAsyncHelmInstallRequest", "cdWfrId", cdWfr.Id, "err", err) + return + } + // build merged values and save PCO history for the release + valuesOverrideResponse, builtChartPath, err := impl.manifestCreationService.BuildManifestForTrigger(overrideRequest, CDAsyncInstallNatsMessage.TriggeredAt, ctx) + if err != nil { + return + } + + _, span := otel.Tracer("orchestrator").Start(ctx, "appService.TriggerRelease") + releaseId, _, releaseErr := impl.cdTriggerService.TriggerRelease(overrideRequest, valuesOverrideResponse, builtChartPath, ctx, CDAsyncInstallNatsMessage.TriggeredAt, CDAsyncInstallNatsMessage.TriggeredBy) + span.End() + if releaseErr != nil { + impl.handleAsyncTriggerReleaseError(releaseErr, cdWfr, overrideRequest, appIdentifier) + } else { + impl.logger.Infow("pipeline triggered successfully !!", "cdPipelineId", overrideRequest.PipelineId, "artifactId", overrideRequest.CiArtifactId, "releaseId", releaseId) + // Update previous deployment runner status (in transaction): Failed + _, span = otel.Tracer("orchestrator").Start(ctx, "updatePreviousDeploymentStatus") + err1 := impl.cdWorkflowCommonService.UpdatePreviousDeploymentStatus(cdWfr, overrideRequest.PipelineId, CDAsyncInstallNatsMessage.TriggeredAt, overrideRequest.UserId) + span.End() + if err1 != nil { + impl.logger.Errorw("error while update previous cd workflow runners, processDevtronAsyncHelmInstallRequest", "err", err, "runner", cdWfr, "pipelineId", overrideRequest.PipelineId) + return + } + } +} + +func (impl *WorkflowDagExecutorImpl) SubscribeDevtronAsyncHelmInstallRequest() error { + callback := func(msg *model.PubSubMsg) { + CDAsyncInstallNatsMessage, appIdentifier, err := impl.extractOverrideRequestFromCDAsyncInstallEvent(msg) + if err != nil { + impl.logger.Errorw("err on extracting override request, SubscribeDevtronAsyncHelmInstallRequest", "err", err) + return + } + if skip := impl.handleConcurrentRequest(CDAsyncInstallNatsMessage.ValuesOverrideRequest.WfrId); skip { + impl.logger.Warnw("concurrent request received, SubscribeDevtronAsyncHelmInstallRequest", "WfrId", CDAsyncInstallNatsMessage.ValuesOverrideRequest.WfrId) + return + } + defer impl.cleanUpDevtronAsyncHelmInstallRequest(CDAsyncInstallNatsMessage.ValuesOverrideRequest.PipelineId, CDAsyncInstallNatsMessage.ValuesOverrideRequest.WfrId) + impl.processDevtronAsyncHelmInstallRequest(CDAsyncInstallNatsMessage, appIdentifier) + return + } + + // add required logging here + var loggerFunc pubsub.LoggerFunc = func(msg model.PubSubMsg) (string, []interface{}) { + CDAsyncInstallNatsMessage := &bean7.AsyncCdDeployEvent{} + err := json.Unmarshal([]byte(msg.Data), CDAsyncInstallNatsMessage) + if err != nil { + return "error in unmarshalling CD async install request nats message", []interface{}{"err", err} + } + return "got message for devtron chart install", []interface{}{"appId", CDAsyncInstallNatsMessage.ValuesOverrideRequest.AppId, "pipelineId", CDAsyncInstallNatsMessage.ValuesOverrideRequest.PipelineId, "artifactId", CDAsyncInstallNatsMessage.ValuesOverrideRequest.CiArtifactId} + } + + err := impl.pubsubClient.Subscribe(pubsub.DEVTRON_CHART_INSTALL_TOPIC, callback, loggerFunc) + if err != nil { + impl.logger.Error(err) + return err + } + return nil +} + +func (impl *WorkflowDagExecutorImpl) handleCiSuccessEvent(triggerContext bean5.TriggerContext, artifact *repository.CiArtifact, async bool, triggeredBy int32) error { + //1. get cd pipelines + //2. get config + //3. trigger wf/ deployment + var pipelineID int + if artifact.DataSource == repository.POST_CI { + pipelineID = artifact.ComponentId + } else { + // TODO: need to migrate artifact.PipelineId for dataSource="CI_RUNNER" also to component_id + pipelineID = artifact.PipelineId + } + pipelines, err := impl.pipelineRepository.FindByParentCiPipelineId(pipelineID) + if err != nil { + impl.logger.Errorw("error in fetching cd pipeline", "pipelineId", artifact.PipelineId, "err", err) + return err + } + for _, pipeline := range pipelines { + triggerRequest := bean5.TriggerRequest{ + CdWf: nil, + Pipeline: pipeline, + Artifact: artifact, + TriggeredBy: triggeredBy, + TriggerContext: triggerContext, + } + err = impl.triggerIfAutoStageCdPipeline(triggerRequest) + if err != nil { + impl.logger.Debugw("error on trigger cd pipeline", "err", err) + } + } + return nil +} + +func (impl *WorkflowDagExecutorImpl) handleWebhookExternalCiEvent(artifact *repository.CiArtifact, triggeredBy int32, externalCiId int, auth func(token string, projectObject string, envObject string) bool, token string) (bool, error) { + hasAnyTriggered := false + appWorkflowMappings, err := impl.appWorkflowRepository.FindWFCDMappingByExternalCiId(externalCiId) + if err != nil { + impl.logger.Errorw("error in fetching cd pipeline", "pipelineId", artifact.PipelineId, "err", err) + return hasAnyTriggered, err + } + + var pipelines []*pipelineConfig.Pipeline + for _, appWorkflowMapping := range appWorkflowMappings { + pipeline, err := impl.pipelineRepository.FindById(appWorkflowMapping.ComponentId) + if err != nil { + impl.logger.Errorw("error in fetching cd pipeline", "pipelineId", artifact.PipelineId, "err", err) + return hasAnyTriggered, err + } + projectObject := impl.enforcerUtil.GetAppRBACNameByAppId(pipeline.AppId) + envObject := impl.enforcerUtil.GetAppRBACByAppIdAndPipelineId(pipeline.AppId, pipeline.Id) + if !auth(token, projectObject, envObject) { + err = &util.ApiError{Code: "401", HttpStatusCode: 401, UserMessage: "Unauthorized"} + return hasAnyTriggered, err + } + if pipeline.TriggerType == pipelineConfig.TRIGGER_TYPE_MANUAL { + impl.logger.Warnw("skipping deployment for manual trigger for webhook", "pipeline", pipeline) + continue + } + pipelines = append(pipelines, pipeline) + } + + for _, pipeline := range pipelines { + //applyAuth=false, already auth applied for this flow + triggerRequest := bean5.TriggerRequest{ + CdWf: nil, + Pipeline: pipeline, + Artifact: artifact, + ApplyAuth: false, + TriggeredBy: triggeredBy, + } + err = impl.triggerIfAutoStageCdPipeline(triggerRequest) + if err != nil { + impl.logger.Debugw("error on trigger cd pipeline", "err", err) + return hasAnyTriggered, err + } + hasAnyTriggered = true + } + + return hasAnyTriggered, err +} + +// if stage is present with 0 stage steps, delete the stage +// handle corrupt data (https://github.com/devtron-labs/devtron/issues/3826) +func (impl *WorkflowDagExecutorImpl) deleteCorruptedPipelineStage(pipelineStage *repository4.PipelineStage, triggeredBy int32) (error, bool) { + if pipelineStage != nil { + stageReq := &bean3.PipelineStageDto{ + Id: pipelineStage.Id, + Type: pipelineStage.Type, + } + err, deleted := impl.pipelineStageService.DeletePipelineStageIfReq(stageReq, triggeredBy) + if err != nil { + impl.logger.Errorw("error in deleting the corrupted pipeline stage", "err", err, "pipelineStageReq", stageReq) + return err, false + } + return nil, deleted + } + return nil, false +} + +func (impl *WorkflowDagExecutorImpl) triggerIfAutoStageCdPipeline(request bean5.TriggerRequest) error { + + preStage, err := impl.getPipelineStage(request.Pipeline.Id, repository4.PIPELINE_STAGE_TYPE_PRE_CD) + if err != nil { + return err + } + + //handle corrupt data (https://github.com/devtron-labs/devtron/issues/3826) + err, deleted := impl.deleteCorruptedPipelineStage(preStage, request.TriggeredBy) + if err != nil { + impl.logger.Errorw("error in deleteCorruptedPipelineStage ", "cdPipelineId", request.Pipeline.Id, "err", err, "preStage", preStage, "triggeredBy", request.TriggeredBy) + return err + } + + request.TriggerContext.Context = context.Background() + if len(request.Pipeline.PreStageConfig) > 0 || (preStage != nil && !deleted) { + // pre stage exists + if request.Pipeline.PreTriggerType == pipelineConfig.TRIGGER_TYPE_AUTOMATIC { + impl.logger.Debugw("trigger pre stage for pipeline", "artifactId", request.Artifact.Id, "pipelineId", request.Pipeline.Id) + err = impl.cdTriggerService.TriggerPreStage(request) // TODO handle error here + return err + } + } else if request.Pipeline.TriggerType == pipelineConfig.TRIGGER_TYPE_AUTOMATIC { + // trigger deployment + impl.logger.Debugw("trigger cd for pipeline", "artifactId", request.Artifact.Id, "pipelineId", request.Pipeline.Id) + err = impl.cdTriggerService.TriggerAutomaticDeployment(request) + return err + } + return nil +} + +func (impl *WorkflowDagExecutorImpl) getPipelineStage(pipelineId int, stageType repository4.PipelineStageType) (*repository4.PipelineStage, error) { + stage, err := impl.pipelineStageService.GetCdStageByCdPipelineIdAndStageType(pipelineId, stageType) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in fetching CD pipeline stage", "cdPipelineId", pipelineId, "stage ", stage, "err", err) + return nil, err + } + return stage, nil +} + +func (impl *WorkflowDagExecutorImpl) HandlePreStageSuccessEvent(triggerContext bean5.TriggerContext, cdStageCompleteEvent bean7.CdStageCompleteEvent) error { + wfRunner, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(cdStageCompleteEvent.WorkflowRunnerId) + if err != nil { + return err + } + if wfRunner.WorkflowType == bean.CD_WORKFLOW_TYPE_PRE { + pipeline, err := impl.pipelineRepository.FindById(cdStageCompleteEvent.CdPipelineId) + if err != nil { + return err + } + ciArtifact, err := impl.ciArtifactRepository.Get(cdStageCompleteEvent.CiArtifactDTO.Id) + if err != nil { + return err + } + // Migration of deprecated DataSource Type + if ciArtifact.IsMigrationRequired() { + migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(ciArtifact.Id) + if migrationErr != nil { + impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", ciArtifact.Id) + } + } + PreCDArtifacts, err := impl.commonArtifactService.SavePluginArtifacts(ciArtifact, cdStageCompleteEvent.PluginRegistryArtifactDetails, pipeline.Id, repository.PRE_CD, cdStageCompleteEvent.TriggeredBy) + if err != nil { + impl.logger.Errorw("error in saving plugin artifacts", "err", err) + return err + } + if pipeline.TriggerType == pipelineConfig.TRIGGER_TYPE_AUTOMATIC { + if len(PreCDArtifacts) > 0 { + ciArtifact = PreCDArtifacts[0] // deployment will be trigger with artifact copied by plugin + } + cdWorkflow, err := impl.cdWorkflowRepository.FindById(cdStageCompleteEvent.WorkflowId) + if err != nil { + return err + } + //passing applyAuth as false since this event is for auto trigger and user who already has access to this cd can trigger pre cd also + applyAuth := false + if cdStageCompleteEvent.TriggeredBy != 1 { + applyAuth = true + } + triggerRequest := bean5.TriggerRequest{ + CdWf: cdWorkflow, + Pipeline: pipeline, + Artifact: ciArtifact, + ApplyAuth: applyAuth, + TriggeredBy: cdStageCompleteEvent.TriggeredBy, + TriggerContext: triggerContext, + } + triggerRequest.TriggerContext.Context = context.Background() + err = impl.cdTriggerService.TriggerAutomaticDeployment(triggerRequest) + if err != nil { + return err + } + } + } + return nil +} + +func (impl *WorkflowDagExecutorImpl) HandleDeploymentSuccessEvent(triggerContext bean5.TriggerContext, pipelineOverride *chartConfig.PipelineOverride) error { + if pipelineOverride == nil { + return fmt.Errorf("invalid request, pipeline override not found") + } + cdWorkflow, err := impl.cdWorkflowRepository.FindById(pipelineOverride.CdWorkflowId) + if err != nil { + impl.logger.Errorw("error in fetching cd workflow by id", "pipelineOverride", pipelineOverride) + return err + } + + postStage, err := impl.getPipelineStage(pipelineOverride.PipelineId, repository4.PIPELINE_STAGE_TYPE_POST_CD) + if err != nil { + return err + } + + var triggeredByUser int32 = 1 + //handle corrupt data (https://github.com/devtron-labs/devtron/issues/3826) + err, deleted := impl.deleteCorruptedPipelineStage(postStage, triggeredByUser) + if err != nil { + impl.logger.Errorw("error in deleteCorruptedPipelineStage ", "err", err, "preStage", postStage, "triggeredBy", triggeredByUser) + return err + } + + if len(pipelineOverride.Pipeline.PostStageConfig) > 0 || (postStage != nil && !deleted) { + if pipelineOverride.Pipeline.PostTriggerType == pipelineConfig.TRIGGER_TYPE_AUTOMATIC && + pipelineOverride.DeploymentType != models.DEPLOYMENTTYPE_STOP && + pipelineOverride.DeploymentType != models.DEPLOYMENTTYPE_START { + + triggerRequest := bean5.TriggerRequest{ + CdWf: cdWorkflow, + Pipeline: pipelineOverride.Pipeline, + TriggeredBy: triggeredByUser, + TriggerContext: triggerContext, + RefCdWorkflowRunnerId: 0, + } + triggerRequest.TriggerContext.Context = context.Background() + err = impl.cdTriggerService.TriggerPostStage(triggerRequest) + if err != nil { + impl.logger.Errorw("error in triggering post stage after successful deployment event", "err", err, "cdWorkflow", cdWorkflow) + return err + } + } + } else { + // to trigger next pre/cd, if any + // finding children cd by pipeline id + err = impl.HandlePostStageSuccessEvent(triggerContext, cdWorkflow.Id, pipelineOverride.PipelineId, 1, nil) + if err != nil { + impl.logger.Errorw("error in triggering children cd after successful deployment event", "parentCdPipelineId", pipelineOverride.PipelineId) + return err + } + } + return nil +} + +func (impl *WorkflowDagExecutorImpl) HandlePostStageSuccessEvent(triggerContext bean5.TriggerContext, cdWorkflowId int, cdPipelineId int, triggeredBy int32, pluginRegistryImageDetails map[string][]string) error { + // finding children cd by pipeline id + cdPipelinesMapping, err := impl.appWorkflowRepository.FindWFCDMappingByParentCDPipelineId(cdPipelineId) + if err != nil { + impl.logger.Errorw("error in getting mapping of cd pipelines by parent cd pipeline id", "err", err, "parentCdPipelineId", cdPipelineId) + return err + } + ciArtifact, err := impl.ciArtifactRepository.GetArtifactByCdWorkflowId(cdWorkflowId) + if err != nil { + impl.logger.Errorw("error in finding artifact by cd workflow id", "err", err, "cdWorkflowId", cdWorkflowId) + return err + } + if len(pluginRegistryImageDetails) > 0 { + PostCDArtifacts, err := impl.commonArtifactService.SavePluginArtifacts(ciArtifact, pluginRegistryImageDetails, cdPipelineId, repository.POST_CD, triggeredBy) + if err != nil { + impl.logger.Errorw("error in saving plugin artifacts", "err", err) + return err + } + if len(PostCDArtifacts) > 0 { + ciArtifact = PostCDArtifacts[0] + } + } + for _, cdPipelineMapping := range cdPipelinesMapping { + //find pipeline by cdPipeline ID + pipeline, err := impl.pipelineRepository.FindById(cdPipelineMapping.ComponentId) + if err != nil { + impl.logger.Errorw("error in getting cd pipeline by id", "err", err, "pipelineId", cdPipelineMapping.ComponentId) + return err + } + //finding ci artifact by ciPipelineID and pipelineId + //TODO : confirm values for applyAuth, async & triggeredBy + + triggerRequest := bean5.TriggerRequest{ + CdWf: nil, + Pipeline: pipeline, + Artifact: ciArtifact, + TriggeredBy: triggeredBy, + TriggerContext: triggerContext, + } + + err = impl.triggerIfAutoStageCdPipeline(triggerRequest) + if err != nil { + impl.logger.Errorw("error in triggering cd pipeline after successful post stage", "err", err, "pipelineId", pipeline.Id) + return err + } + } + return nil +} + +func (impl *WorkflowDagExecutorImpl) HandleCiSuccessEvent(triggerContext bean5.TriggerContext, ciPipelineId int, request *bean2.CiArtifactWebhookRequest, imagePushedAt *time.Time) (id int, err error) { + impl.logger.Infow("webhook for artifact save", "req", request) + if request.WorkflowId != nil { + savedWorkflow, err := impl.ciWorkflowRepository.FindById(*request.WorkflowId) + if err != nil { + impl.logger.Errorw("cannot get saved wf", "err", err) + return 0, err + } + // if workflow already cancelled then return, this state arises when user force aborts a ci + if savedWorkflow.Status == executors.WorkflowCancel { + return 0, err + } + savedWorkflow.Status = string(v1alpha1.NodeSucceeded) + impl.logger.Debugw("updating workflow ", "savedWorkflow", savedWorkflow) + err = impl.ciWorkflowRepository.UpdateWorkFlow(savedWorkflow) + if err != nil { + impl.logger.Errorw("update wf failed for id ", "err", err) + return 0, err + } + } + + pipeline, err := impl.ciPipelineRepository.FindByCiAndAppDetailsById(ciPipelineId) + if request.PipelineName == "" { + request.PipelineName = pipeline.Name + } + if err != nil { + impl.logger.Errorw("unable to find pipeline", "name", request.PipelineName, "err", err) + return 0, err + } + materialJson, err := request.MaterialInfo.MarshalJSON() + if err != nil { + impl.logger.Errorw("unable to marshal material metadata", "err", err) + return 0, err + } + dst := new(bytes.Buffer) + err = json.Compact(dst, materialJson) + if err != nil { + return 0, err + } + materialJson = dst.Bytes() + createdOn := time.Now() + updatedOn := time.Now() + if !imagePushedAt.IsZero() { + createdOn = *imagePushedAt + } + buildArtifact := &repository.CiArtifact{ + Image: request.Image, + ImageDigest: request.ImageDigest, + MaterialInfo: string(materialJson), + DataSource: request.DataSource, + PipelineId: pipeline.Id, + WorkflowId: request.WorkflowId, + ScanEnabled: pipeline.ScanEnabled, + Scanned: false, + IsArtifactUploaded: request.IsArtifactUploaded, + AuditLog: sql.AuditLog{CreatedBy: request.UserId, UpdatedBy: request.UserId, CreatedOn: createdOn, UpdatedOn: updatedOn}, + } + plugin, err := impl.globalPluginRepository.GetPluginByName(bean3.VULNERABILITY_SCANNING_PLUGIN) + if err != nil || len(plugin) == 0 { + impl.logger.Errorw("error in getting image scanning plugin", "err", err) + return 0, err + } + isScanPluginConfigured, err := impl.pipelineStageRepository.CheckPluginExistsInCiPipeline(pipeline.Id, string(repository4.PIPELINE_STAGE_TYPE_POST_CI), plugin[0].Id) + if err != nil { + impl.logger.Errorw("error in getting ci pipeline plugin", "err", err, "pipelineId", pipeline.Id, "pluginId", plugin[0].Id) + return 0, err + } + if pipeline.ScanEnabled || isScanPluginConfigured { + buildArtifact.Scanned = true + buildArtifact.ScanEnabled = true + } + if err = impl.ciArtifactRepository.Save(buildArtifact); err != nil { + impl.logger.Errorw("error in saving material", "err", err) + return 0, err + } + + var pluginArtifacts []*repository.CiArtifact + for registry, artifacts := range request.PluginRegistryArtifactDetails { + for _, image := range artifacts { + if pipeline.PipelineType == bean3.CI_JOB && image == "" { + continue + } + pluginArtifact := &repository.CiArtifact{ + Image: image, + ImageDigest: request.ImageDigest, + MaterialInfo: string(materialJson), + DataSource: request.PluginArtifactStage, + ComponentId: pipeline.Id, + PipelineId: pipeline.Id, + AuditLog: sql.AuditLog{CreatedBy: request.UserId, UpdatedBy: request.UserId, CreatedOn: createdOn, UpdatedOn: updatedOn}, + CredentialsSourceType: repository.GLOBAL_CONTAINER_REGISTRY, + CredentialSourceValue: registry, + ParentCiArtifact: buildArtifact.Id, + Scanned: buildArtifact.Scanned, + ScanEnabled: buildArtifact.ScanEnabled, + } + pluginArtifacts = append(pluginArtifacts, pluginArtifact) + } + } + if len(pluginArtifacts) > 0 { + _, err = impl.ciArtifactRepository.SaveAll(pluginArtifacts) + if err != nil { + impl.logger.Errorw("error while saving ci artifacts", "err", err) + return 0, err + } + } + + childrenCi, err := impl.ciPipelineRepository.FindByParentCiPipelineId(ciPipelineId) + if err != nil && !util.IsErrNoRows(err) { + impl.logger.Errorw("error while fetching childern ci ", "err", err) + return 0, err + } + + var ciArtifactArr []*repository.CiArtifact + for _, ci := range childrenCi { + ciArtifact := &repository.CiArtifact{ + Image: request.Image, + ImageDigest: request.ImageDigest, + MaterialInfo: string(materialJson), + DataSource: request.DataSource, + PipelineId: ci.Id, + ParentCiArtifact: buildArtifact.Id, + ScanEnabled: ci.ScanEnabled, + Scanned: false, + IsArtifactUploaded: request.IsArtifactUploaded, + AuditLog: sql.AuditLog{CreatedBy: request.UserId, UpdatedBy: request.UserId, CreatedOn: time.Now(), UpdatedOn: time.Now()}, + } + if ci.ScanEnabled { + ciArtifact.Scanned = true + } + ciArtifactArr = append(ciArtifactArr, ciArtifact) + } + + impl.logger.Debugw("saving ci artifacts", "art", ciArtifactArr) + if len(ciArtifactArr) > 0 { + _, err = impl.ciArtifactRepository.SaveAll(ciArtifactArr) + if err != nil { + impl.logger.Errorw("error while saving ci artifacts", "err", err) + return 0, err + } + } + if len(pluginArtifacts) == 0 { + ciArtifactArr = append(ciArtifactArr, buildArtifact) + } else { + ciArtifactArr = append(ciArtifactArr, pluginArtifacts[0]) + } + go impl.WriteCISuccessEvent(request, pipeline, buildArtifact) + async := false + + // execute auto trigger in batch on CI success event + totalCIArtifactCount := len(ciArtifactArr) + batchSize := impl.ciConfig.CIAutoTriggerBatchSize + // handling to avoid infinite loop + if batchSize <= 0 { + batchSize = 1 + } + start := time.Now() + impl.logger.Infow("Started: auto trigger for children Stage/CD pipelines", "Artifact count", totalCIArtifactCount) + for i := 0; i < totalCIArtifactCount; { + // requests left to process + remainingBatch := totalCIArtifactCount - i + if remainingBatch < batchSize { + batchSize = remainingBatch + } + var wg sync.WaitGroup + for j := 0; j < batchSize; j++ { + wg.Add(1) + index := i + j + go func(index int) { + defer wg.Done() + ciArtifact := ciArtifactArr[index] + // handle individual CiArtifact success event + err = impl.handleCiSuccessEvent(triggerContext, ciArtifact, async, request.UserId) + if err != nil { + impl.logger.Errorw("error on handle ci success event", "ciArtifactId", ciArtifact.Id, "err", err) + } + }(index) + } + wg.Wait() + i += batchSize + } + impl.logger.Debugw("Completed: auto trigger for children Stage/CD pipelines", "Time taken", time.Since(start).Seconds()) + return buildArtifact.Id, err +} + +func (impl *WorkflowDagExecutorImpl) WriteCISuccessEvent(request *bean2.CiArtifactWebhookRequest, pipeline *pipelineConfig.CiPipeline, artifact *repository.CiArtifact) { + event := impl.eventFactory.Build(util2.Success, &pipeline.Id, pipeline.AppId, nil, util2.CI) + event.CiArtifactId = artifact.Id + if artifact.WorkflowId != nil { + event.CiWorkflowRunnerId = *artifact.WorkflowId + } + event.UserId = int(request.UserId) + event = impl.eventFactory.BuildExtraCIData(event, nil, artifact.Image) + _, evtErr := impl.eventClient.WriteNotificationEvent(event) + if evtErr != nil { + impl.logger.Errorw("error in writing event", "err", evtErr) + } +} + +func (impl *WorkflowDagExecutorImpl) HandleCiStepFailedEvent(ciPipelineId int, request *bean2.CiArtifactWebhookRequest) (err error) { + + savedWorkflow, err := impl.ciWorkflowRepository.FindById(*request.WorkflowId) + if err != nil { + impl.logger.Errorw("cannot get saved wf", "wf ID: ", *request.WorkflowId, "err", err) + return err + } + + pipeline, err := impl.ciPipelineRepository.FindByCiAndAppDetailsById(ciPipelineId) + if err != nil { + impl.logger.Errorw("unable to find pipeline", "ID", ciPipelineId, "err", err) + return err + } + + go func() { + if len(savedWorkflow.ImagePathReservationIds) > 0 { + err = impl.customTagService.DeactivateImagePathReservationByImageIds(savedWorkflow.ImagePathReservationIds) + if err != nil { + impl.logger.Errorw("unable to deactivate impage_path_reservation ", err) + } + } + }() + + go impl.WriteCIStepFailedEvent(pipeline, request, savedWorkflow) + return nil +} + +func (impl *WorkflowDagExecutorImpl) WriteCIStepFailedEvent(pipeline *pipelineConfig.CiPipeline, request *bean2.CiArtifactWebhookRequest, ciWorkflow *pipelineConfig.CiWorkflow) { + event := impl.eventFactory.Build(util2.Fail, &pipeline.Id, pipeline.AppId, nil, util2.CI) + material := &client.MaterialTriggerInfo{} + material.GitTriggers = ciWorkflow.GitTriggers + event.CiWorkflowRunnerId = ciWorkflow.Id + event.UserId = int(ciWorkflow.TriggeredBy) + event = impl.eventFactory.BuildExtraCIData(event, material, request.Image) + event.CiArtifactId = 0 + event.Payload.FailureReason = request.FailureReason + _, evtErr := impl.eventClient.WriteNotificationEvent(event) + if evtErr != nil { + impl.logger.Errorw("error in writing event: ", event, "error: ", evtErr) + } +} + +func (impl *WorkflowDagExecutorImpl) HandleExternalCiWebhook(externalCiId int, request *bean2.CiArtifactWebhookRequest, + auth func(token string, projectObject string, envObject string) bool, token string) (id int, err error) { + externalCiPipeline, err := impl.ciPipelineRepository.FindExternalCiById(externalCiId) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in fetching external ci", "err", err) + return 0, err + } + if externalCiPipeline.Id == 0 { + impl.logger.Errorw("invalid external ci id", "externalCiId", externalCiId, "err", err) + return 0, &util.ApiError{Code: "400", HttpStatusCode: 400, UserMessage: "invalid external ci id"} + } + + impl.logger.Infow("request of webhook external ci", "req", request) + materialJson, err := request.MaterialInfo.MarshalJSON() + if err != nil { + impl.logger.Errorw("unable to marshal material metadata", "err", err) + return 0, err + } + dst := new(bytes.Buffer) + err = json.Compact(dst, materialJson) + if err != nil { + impl.logger.Errorw("parsing error", "err", err) + return 0, err + } + materialJson = dst.Bytes() + artifact := &repository.CiArtifact{ + Image: request.Image, + ImageDigest: request.ImageDigest, + MaterialInfo: string(materialJson), + DataSource: request.DataSource, + WorkflowId: request.WorkflowId, + ExternalCiPipelineId: externalCiId, + ScanEnabled: false, + Scanned: false, + IsArtifactUploaded: request.IsArtifactUploaded, + AuditLog: sql.AuditLog{CreatedBy: request.UserId, UpdatedBy: request.UserId, CreatedOn: time.Now(), UpdatedOn: time.Now()}, + } + if err = impl.ciArtifactRepository.Save(artifact); err != nil { + impl.logger.Errorw("error in saving material", "err", err) + return 0, err + } + + hasAnyTriggered, err := impl.handleWebhookExternalCiEvent(artifact, request.UserId, externalCiId, auth, token) + if err != nil { + impl.logger.Errorw("error on handle ext ci webhook", "err", err) + // if none of the child node has been triggered + if !hasAnyTriggered { + if err1 := impl.ciArtifactRepository.Delete(artifact); err1 != nil { + impl.logger.Errorw("error in rollback artifact", "err", err1) + return 0, err1 + } + } + } + return artifact.Id, err +} + +// TODO: move in adapter +func (impl *WorkflowDagExecutorImpl) BuildCiArtifactRequestForWebhook(event pipeline.ExternalCiWebhookDto) (*bean2.CiArtifactWebhookRequest, error) { + ciMaterialInfos := make([]repository.CiMaterialInfo, 0) + if event.MaterialType == "" { + event.MaterialType = "git" + } + for _, p := range event.CiProjectDetails { + var modifications []repository.Modification + + var branch string + var tag string + var webhookData repository.WebhookData + if p.SourceType == pipelineConfig.SOURCE_TYPE_BRANCH_FIXED { + branch = p.SourceValue + } else if p.SourceType == pipelineConfig.SOURCE_TYPE_WEBHOOK { + webhookData = repository.WebhookData{ + Id: p.WebhookData.Id, + EventActionType: p.WebhookData.EventActionType, + Data: p.WebhookData.Data, + } + } + + modification := repository.Modification{ + Revision: p.CommitHash, + ModifiedTime: p.CommitTime, + Author: p.Author, + Branch: branch, + Tag: tag, + WebhookData: webhookData, + Message: p.Message, + } + + modifications = append(modifications, modification) + ciMaterialInfo := repository.CiMaterialInfo{ + Material: repository.Material{ + GitConfiguration: repository.GitConfiguration{ + URL: p.GitRepository, + }, + Type: event.MaterialType, + }, + Changed: true, + Modifications: modifications, + } + ciMaterialInfos = append(ciMaterialInfos, ciMaterialInfo) + } + + materialBytes, err := json.Marshal(ciMaterialInfos) + if err != nil { + impl.logger.Errorw("cannot build ci artifact req", "err", err) + return nil, err + } + rawMaterialInfo := json.RawMessage(materialBytes) + fmt.Printf("Raw Message : %s\n", rawMaterialInfo) + + if event.TriggeredBy == 0 { + event.TriggeredBy = 1 // system triggered event + } + + request := &bean2.CiArtifactWebhookRequest{ + Image: event.DockerImage, + ImageDigest: event.Digest, + DataSource: event.DataSource, + PipelineName: event.PipelineName, + MaterialInfo: rawMaterialInfo, + UserId: event.TriggeredBy, + WorkflowId: event.WorkflowId, + IsArtifactUploaded: event.IsArtifactUploaded, + } + // if DataSource is empty, repository.WEBHOOK is considered as default + if request.DataSource == "" { + request.DataSource = repository.WEBHOOK + } + return request, nil +} diff --git a/pkg/workflow/dag/bean/bean.go b/pkg/workflow/dag/bean/bean.go new file mode 100644 index 0000000000..4e4af68cdb --- /dev/null +++ b/pkg/workflow/dag/bean/bean.go @@ -0,0 +1,20 @@ +package bean + +import ( + "encoding/json" + "github.com/devtron-labs/devtron/internal/sql/repository" +) + +type CiArtifactWebhookRequest struct { + Image string `json:"image" validate:"required"` + ImageDigest string `json:"imageDigest"` + MaterialInfo json.RawMessage `json:"materialInfo"` + DataSource repository.ArtifactsSourceType `json:"dataSource" validate:"oneof=CI-RUNNER EXTERNAL pre_cd post_cd post_ci GOCD"` + PipelineName string `json:"pipelineName"` + WorkflowId *int `json:"workflowId"` + UserId int32 `json:"userId"` + IsArtifactUploaded bool `json:"isArtifactUploaded"` + FailureReason string `json:"failureReason"` + PluginRegistryArtifactDetails map[string][]string `json:"PluginRegistryArtifactDetails"` //map of registry and array of images generated by Copy container image plugin + PluginArtifactStage string `json:"pluginArtifactStage"` // at which stage of CI artifact was generated by plugin ("pre_ci/post_ci") +} diff --git a/pkg/workflow/status/WorkflowStatusService.go b/pkg/workflow/status/WorkflowStatusService.go new file mode 100644 index 0000000000..05534b5d9d --- /dev/null +++ b/pkg/workflow/status/WorkflowStatusService.go @@ -0,0 +1,613 @@ +package status + +import ( + "context" + "fmt" + application2 "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" + pubub "github.com/devtron-labs/common-lib/pubsub-lib" + bean2 "github.com/devtron-labs/devtron/api/bean" + client "github.com/devtron-labs/devtron/api/helm-app/service" + "github.com/devtron-labs/devtron/client/argocdServer" + "github.com/devtron-labs/devtron/client/argocdServer/application" + client2 "github.com/devtron-labs/devtron/client/events" + appRepository "github.com/devtron-labs/devtron/internal/sql/repository/app" + "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/pkg/app" + "github.com/devtron-labs/devtron/pkg/app/status" + app_status "github.com/devtron-labs/devtron/pkg/appStatus" + repository3 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" + repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" + bean3 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/types" + "github.com/devtron-labs/devtron/pkg/sql" + "github.com/devtron-labs/devtron/pkg/workflow/dag" + "github.com/devtron-labs/devtron/pkg/workflow/status/bean" + util3 "github.com/devtron-labs/devtron/util" + "github.com/devtron-labs/devtron/util/argo" + "github.com/go-pg/pg" + "go.uber.org/zap" + "slices" + "time" +) + +type WorkflowStatusService interface { + CheckHelmAppStatusPeriodicallyAndUpdateInDb(helmPipelineStatusCheckEligibleTime int, + getPipelineDeployedWithinHours int) error + + UpdatePipelineTimelineAndStatusByLiveApplicationFetch(triggerContext bean3.TriggerContext, + pipeline *pipelineConfig.Pipeline, installedApp repository3.InstalledApps, userId int32) (error, bool) + + CheckAndSendArgoPipelineStatusSyncEventIfNeeded(pipelineId int, installedAppVersionId int, + userId int32, isAppStoreApplication bool) + + CheckArgoPipelineTimelineStatusPeriodicallyAndUpdateInDb(pendingSinceSeconds int, timeForDegradation int) error + + CheckArgoAppStatusPeriodicallyAndUpdateInDb(getPipelineDeployedBeforeMinutes int, getPipelineDeployedWithinHours int) error +} + +type WorkflowStatusServiceImpl struct { + logger *zap.SugaredLogger + workflowDagExecutor dag.WorkflowDagExecutor + pipelineStatusTimelineService status.PipelineStatusTimelineService + appService app.AppService + config *types.CdConfig + appStatusService app_status.AppStatusService + acdConfig *argocdServer.ACDConfig + AppConfig *app.AppServiceConfig + argoUserService argo.ArgoUserService + pipelineStatusSyncDetailService status.PipelineStatusSyncDetailService + argocdClientWrapperService argocdServer.ArgoClientWrapperService + + cdWorkflowRepository pipelineConfig.CdWorkflowRepository + pipelineOverrideRepository chartConfig.PipelineOverrideRepository + installedAppVersionHistoryRepository repository3.InstalledAppVersionHistoryRepository + appRepository appRepository.AppRepository + envRepository repository2.EnvironmentRepository + installedAppRepository repository3.InstalledAppRepository + pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository + pipelineRepository pipelineConfig.PipelineRepository + + application application.ServiceClient + eventClient client2.EventClient +} + +func NewWorkflowStatusServiceImpl(logger *zap.SugaredLogger, + workflowDagExecutor dag.WorkflowDagExecutor, + pipelineStatusTimelineService status.PipelineStatusTimelineService, + appService app.AppService, appStatusService app_status.AppStatusService, + acdConfig *argocdServer.ACDConfig, AppConfig *app.AppServiceConfig, + argoUserService argo.ArgoUserService, + pipelineStatusSyncDetailService status.PipelineStatusSyncDetailService, + argocdClientWrapperService argocdServer.ArgoClientWrapperService, + cdWorkflowRepository pipelineConfig.CdWorkflowRepository, + pipelineOverrideRepository chartConfig.PipelineOverrideRepository, + installedAppVersionHistoryRepository repository3.InstalledAppVersionHistoryRepository, + appRepository appRepository.AppRepository, + envRepository repository2.EnvironmentRepository, + installedAppRepository repository3.InstalledAppRepository, + pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository, + pipelineRepository pipelineConfig.PipelineRepository, + application application.ServiceClient, + eventClient client2.EventClient) (*WorkflowStatusServiceImpl, error) { + impl := &WorkflowStatusServiceImpl{ + logger: logger, + workflowDagExecutor: workflowDagExecutor, + pipelineStatusTimelineService: pipelineStatusTimelineService, + appService: appService, + appStatusService: appStatusService, + acdConfig: acdConfig, + AppConfig: AppConfig, + argoUserService: argoUserService, + pipelineStatusSyncDetailService: pipelineStatusSyncDetailService, + argocdClientWrapperService: argocdClientWrapperService, + cdWorkflowRepository: cdWorkflowRepository, + pipelineOverrideRepository: pipelineOverrideRepository, + installedAppVersionHistoryRepository: installedAppVersionHistoryRepository, + appRepository: appRepository, + envRepository: envRepository, + installedAppRepository: installedAppRepository, + pipelineStatusTimelineRepository: pipelineStatusTimelineRepository, + pipelineRepository: pipelineRepository, + application: application, + eventClient: eventClient, + } + config, err := types.GetCdConfig() + if err != nil { + return nil, err + } + impl.config = config + return impl, nil +} + +func (impl *WorkflowStatusServiceImpl) CheckHelmAppStatusPeriodicallyAndUpdateInDb(helmPipelineStatusCheckEligibleTime int, + getPipelineDeployedWithinHours int) error { + wfrList, err := impl.cdWorkflowRepository.GetLatestTriggersOfHelmPipelinesStuckInNonTerminalStatuses(getPipelineDeployedWithinHours) + if err != nil { + impl.logger.Errorw("error in getting latest triggers of helm pipelines which are stuck in non terminal statuses", "err", err) + return err + } + impl.logger.Debugw("checking helm app status for non terminal deployment triggers", "wfrList", wfrList, "number of wfr", len(wfrList)) + for _, wfr := range wfrList { + if time.Now().Sub(wfr.StartedOn) <= time.Duration(helmPipelineStatusCheckEligibleTime)*time.Second { + // if wfr is updated within configured time then do not include for this cron cycle + continue + } + appIdentifier := &client.AppIdentifier{ + ClusterId: wfr.CdWorkflow.Pipeline.Environment.ClusterId, + Namespace: wfr.CdWorkflow.Pipeline.Environment.Namespace, + ReleaseName: wfr.CdWorkflow.Pipeline.DeploymentAppName, + } + if isWfrUpdated := impl.workflowDagExecutor.UpdateWorkflowRunnerStatusForDeployment(appIdentifier, wfr, true); !isWfrUpdated { + continue + } + wfr.UpdatedBy = 1 + wfr.UpdatedOn = time.Now() + if wfr.Status == pipelineConfig.WorkflowFailed { + err = impl.pipelineStatusTimelineService.MarkPipelineStatusTimelineFailed(wfr.RefCdWorkflowRunnerId, pipelineConfig.NEW_DEPLOYMENT_INITIATED) + if err != nil { + impl.logger.Errorw("error updating CdPipelineStatusTimeline", "err", err) + return err + } + } + err = impl.cdWorkflowRepository.UpdateWorkFlowRunner(wfr) + if err != nil { + impl.logger.Errorw("error on update cd workflow runner", "wfr", wfr, "err", err) + return err + } + if slices.Contains(pipelineConfig.WfrTerminalStatusList, wfr.Status) { + util3.TriggerCDMetrics(pipelineConfig.GetTriggerMetricsFromRunnerObj(wfr), impl.config.ExposeCDMetrics) + } + + impl.logger.Infow("updated workflow runner status for helm app", "wfr", wfr) + if wfr.Status == pipelineConfig.WorkflowSucceeded { + pipelineOverride, err := impl.pipelineOverrideRepository.FindLatestByCdWorkflowId(wfr.CdWorkflowId) + if err != nil { + impl.logger.Errorw("error in getting latest pipeline override by cdWorkflowId", "err", err, "cdWorkflowId", wfr.CdWorkflowId) + return err + } + go impl.appService.WriteCDSuccessEvent(pipelineOverride.Pipeline.AppId, pipelineOverride.Pipeline.EnvironmentId, pipelineOverride) + err = impl.workflowDagExecutor.HandleDeploymentSuccessEvent(bean3.TriggerContext{}, pipelineOverride) + if err != nil { + impl.logger.Errorw("error on handling deployment success event", "wfr", wfr, "err", err) + return err + } + } + } + return nil +} + +func (impl *WorkflowStatusServiceImpl) UpdatePipelineTimelineAndStatusByLiveApplicationFetch(triggerContext bean3.TriggerContext, + pipeline *pipelineConfig.Pipeline, installedApp repository3.InstalledApps, userId int32) (error, bool) { + isTimelineUpdated := false + isSucceeded := false + var pipelineOverride *chartConfig.PipelineOverride + if pipeline != nil { + isAppStore := false + cdWfr, err := impl.cdWorkflowRepository.FindLastStatusByPipelineIdAndRunnerType(pipeline.Id, bean2.CD_WORKFLOW_TYPE_DEPLOY) + if err != nil { + impl.logger.Errorw("error in getting latest cdWfr by cdPipelineId", "err", err, "pipelineId", pipeline.Id) + return nil, isTimelineUpdated + } + impl.logger.Debugw("ARGO_PIPELINE_STATUS_UPDATE_REQ", "stage", "checkingDeploymentStatus", "argoAppName", pipeline, "cdWfr", cdWfr) + if util3.IsTerminalStatus(cdWfr.Status) { + // drop event + return nil, isTimelineUpdated + } + + if !impl.acdConfig.ArgoCDAutoSyncEnabled { + // if manual sync check for application sync status + isArgoAppSynced := impl.pipelineStatusTimelineService.GetArgoAppSyncStatus(cdWfr.Id) + if !isArgoAppSynced { + return nil, isTimelineUpdated + } + } + // this should only be called when we have git-ops configured + // try fetching status from argo cd + acdToken, err := impl.argoUserService.GetLatestDevtronArgoCdUserToken() + if err != nil { + impl.logger.Errorw("error in getting acd token", "err", err) + } + ctx := context.WithValue(context.Background(), "token", acdToken) + query := &application2.ApplicationQuery{ + Name: &pipeline.DeploymentAppName, + } + app, err := impl.application.Get(ctx, query) + if err != nil { + impl.logger.Errorw("error in getting acd application", "err", err, "argoAppName", pipeline) + // updating cdWfr status + cdWfr.Status = pipelineConfig.WorkflowUnableToFetchState + cdWfr.UpdatedOn = time.Now() + cdWfr.UpdatedBy = 1 + err = impl.cdWorkflowRepository.UpdateWorkFlowRunner(&cdWfr) + if err != nil { + impl.logger.Errorw("error on update cd workflow runner", "cdWfr", cdWfr, "err", err) + return err, isTimelineUpdated + } + // creating cd pipeline status timeline + timeline := &pipelineConfig.PipelineStatusTimeline{ + CdWorkflowRunnerId: cdWfr.Id, + Status: pipelineConfig.TIMELINE_STATUS_UNABLE_TO_FETCH_STATUS, + StatusDetail: "Failed to connect to Argo CD to fetch deployment status.", + StatusTime: time.Now(), + AuditLog: sql.AuditLog{ + CreatedBy: userId, + CreatedOn: time.Now(), + UpdatedBy: userId, + UpdatedOn: time.Now(), + }, + } + err = impl.pipelineStatusTimelineService.SaveTimeline(timeline, nil, isAppStore) + if err != nil { + impl.logger.Errorw("error in creating timeline status for app", "err", err, "timeline", timeline) + return err, isTimelineUpdated + } + } else { + if app == nil { + impl.logger.Errorw("found empty argo application object", "appName", pipeline.DeploymentAppName) + return fmt.Errorf("found empty argo application object"), isTimelineUpdated + } + isSucceeded, isTimelineUpdated, pipelineOverride, err = impl.appService.UpdateDeploymentStatusForGitOpsPipelines(app, time.Now(), isAppStore) + if err != nil { + impl.logger.Errorw("error in updating deployment status for gitOps cd pipelines", "app", app) + return err, isTimelineUpdated + } + appStatus := app.Status.Health.Status + err = impl.appStatusService.UpdateStatusWithAppIdEnvId(pipeline.AppId, pipeline.EnvironmentId, string(appStatus)) + if err != nil { + impl.logger.Errorw("error occurred while updating app-status for cd pipeline", "err", err, "appId", pipeline.AppId, "envId", pipeline.EnvironmentId) + impl.logger.Debugw("ignoring the error, UpdateStatusWithAppIdEnvId", "err", err, "appId", pipeline.AppId, "envId", pipeline.EnvironmentId) + } + } + if isSucceeded { + // handling deployment success event + err = impl.workflowDagExecutor.HandleDeploymentSuccessEvent(triggerContext, pipelineOverride) + if err != nil { + impl.logger.Errorw("error in handling deployment success event", "pipelineOverride", pipelineOverride, "err", err) + return err, isTimelineUpdated + } + } + } else { + isAppStore := true + installedAppVersionHistory, err := impl.installedAppVersionHistoryRepository.GetLatestInstalledAppVersionHistoryByInstalledAppId(installedApp.Id) + if err != nil { + impl.logger.Errorw("error in getting latest installedAppVersionHistory by installedAppId", "err", err, "installedAppId", installedApp.Id) + return nil, isTimelineUpdated + } + impl.logger.Debugw("ARGO_PIPELINE_STATUS_UPDATE_REQ", "stage", "checkingDeploymentStatus", "argoAppName", installedApp, "installedAppVersionHistory", installedAppVersionHistory) + if util3.IsTerminalStatus(installedAppVersionHistory.Status) { + // drop event + return nil, isTimelineUpdated + } + if !impl.acdConfig.ArgoCDAutoSyncEnabled { + isArgoAppSynced := impl.pipelineStatusTimelineService.GetArgoAppSyncStatusForAppStore(installedAppVersionHistory.Id) + if !isArgoAppSynced { + return nil, isTimelineUpdated + } + } + appDetails, err := impl.appRepository.FindActiveById(installedApp.AppId) + if err != nil { + impl.logger.Errorw("error in getting appDetails from appId", "err", err) + return nil, isTimelineUpdated + } + // TODO if Environment object in installedApp is nil then fetch envDetails also from envRepository + envDetail, err := impl.envRepository.FindById(installedApp.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in getting envDetails from environment id", "err", err) + return nil, isTimelineUpdated + } + var acdAppName string + if len(installedApp.Environment.Name) != 0 { + acdAppName = appDetails.AppName + installedApp.Environment.Name + } else { + acdAppName = appDetails.AppName + "-" + envDetail.Name + } + + // this should only be called when we have git-ops configured + // try fetching status from argo cd + acdToken, err := impl.argoUserService.GetLatestDevtronArgoCdUserToken() + if err != nil { + impl.logger.Errorw("error in getting acd token", "err", err) + } + + ctx := context.WithValue(context.Background(), "token", acdToken) + query := &application2.ApplicationQuery{ + Name: &acdAppName, + } + app, err := impl.application.Get(ctx, query) + if err != nil { + impl.logger.Errorw("error in getting acd application", "err", err, "argoAppName", installedApp) + // updating cdWfr status + installedAppVersionHistory.Status = pipelineConfig.WorkflowUnableToFetchState + installedAppVersionHistory.UpdatedOn = time.Now() + installedAppVersionHistory.UpdatedBy = 1 + installedAppVersionHistory, err = impl.installedAppVersionHistoryRepository.UpdateInstalledAppVersionHistory(installedAppVersionHistory, nil) + if err != nil { + impl.logger.Errorw("error on update installedAppVersionHistory", "installedAppVersionHistory", installedAppVersionHistory, "err", err) + return err, isTimelineUpdated + } + // creating installedApp pipeline status timeline + timeline := &pipelineConfig.PipelineStatusTimeline{ + InstalledAppVersionHistoryId: installedAppVersionHistory.Id, + Status: pipelineConfig.TIMELINE_STATUS_UNABLE_TO_FETCH_STATUS, + StatusDetail: "Failed to connect to Argo CD to fetch deployment status.", + StatusTime: time.Now(), + AuditLog: sql.AuditLog{ + CreatedBy: userId, + CreatedOn: time.Now(), + UpdatedBy: userId, + UpdatedOn: time.Now(), + }, + } + err = impl.pipelineStatusTimelineService.SaveTimeline(timeline, nil, isAppStore) + if err != nil { + impl.logger.Errorw("error in creating timeline status for app", "err", err, "timeline", timeline) + return err, isTimelineUpdated + } + } else { + if app == nil { + impl.logger.Errorw("found empty argo application object", "appName", acdAppName) + return fmt.Errorf("found empty argo application object"), isTimelineUpdated + } + isSucceeded, isTimelineUpdated, pipelineOverride, err = impl.appService.UpdateDeploymentStatusForGitOpsPipelines(app, time.Now(), isAppStore) + if err != nil { + impl.logger.Errorw("error in updating deployment status for gitOps cd pipelines", "app", app) + return err, isTimelineUpdated + } + appStatus := app.Status.Health.Status + err = impl.appStatusService.UpdateStatusWithAppIdEnvId(installedApp.AppId, installedApp.EnvironmentId, string(appStatus)) + if err != nil { + impl.logger.Errorw("error occurred while updating app-status for installed app", "err", err, "appId", installedApp.AppId, "envId", installedApp.EnvironmentId) + impl.logger.Debugw("ignoring the error, UpdateStatusWithAppIdEnvId", "err", err, "appId", installedApp.AppId, "envId", installedApp.EnvironmentId) + } + } + if isSucceeded { + // handling deployment success event + // updating cdWfr status + installedAppVersionHistory.Status = pipelineConfig.WorkflowSucceeded + installedAppVersionHistory.FinishedOn = time.Now() + installedAppVersionHistory.UpdatedOn = time.Now() + installedAppVersionHistory.UpdatedBy = 1 + installedAppVersionHistory, err = impl.installedAppVersionHistoryRepository.UpdateInstalledAppVersionHistory(installedAppVersionHistory, nil) + if err != nil { + impl.logger.Errorw("error on update installedAppVersionHistory", "installedAppVersionHistory", installedAppVersionHistory, "err", err) + return err, isTimelineUpdated + } + + } + } + + return nil, isTimelineUpdated +} + +func (impl *WorkflowStatusServiceImpl) CheckAndSendArgoPipelineStatusSyncEventIfNeeded(pipelineId int, installedAppVersionId int, + userId int32, isAppStoreApplication bool) { + var lastSyncTime time.Time + var err error + if isAppStoreApplication { + lastSyncTime, err = impl.pipelineStatusSyncDetailService.GetLastSyncTimeForLatestInstalledAppVersionHistoryByInstalledAppVersionId(installedAppVersionId) + } else { + lastSyncTime, err = impl.pipelineStatusSyncDetailService.GetLastSyncTimeForLatestCdWfrByCdPipelineId(pipelineId) + } + if err != nil { + impl.logger.Errorw("error in getting last sync time by pipelineId", "err", err, "pipelineId", pipelineId, "installedAppVersionHistoryId", installedAppVersionId) + return + } + + // sync argocd app + if pipelineId != 0 { + err := impl.syncACDDevtronApps(impl.AppConfig.ArgocdManualSyncCronPipelineDeployedBefore, pipelineId) + if err != nil { + impl.logger.Errorw("error in syncing devtron apps deployed via argoCD", "err", err) + return + } + } + if installedAppVersionId != 0 { + err := impl.syncACDHelmApps(impl.AppConfig.ArgocdManualSyncCronPipelineDeployedBefore, installedAppVersionId) + if err != nil { + impl.logger.Errorw("error in syncing Helm apps deployed via argoCD", "err", err) + return + } + } + + // pipelineId can be cdPipelineId or installedAppVersionId, using isAppStoreApplication flag to identify between them + if lastSyncTime.IsZero() || (!lastSyncTime.IsZero() && time.Since(lastSyncTime) > 5*time.Second) { // create new nats event + statusUpdateEvent := bean.ArgoPipelineStatusSyncEvent{ + PipelineId: pipelineId, + InstalledAppVersionId: installedAppVersionId, + UserId: userId, + IsAppStoreApplication: isAppStoreApplication, + } + // write event + err = impl.eventClient.WriteNatsEvent(pubub.ARGO_PIPELINE_STATUS_UPDATE_TOPIC, statusUpdateEvent) + if err != nil { + impl.logger.Errorw("error in writing nats event", "topic", pubub.ARGO_PIPELINE_STATUS_UPDATE_TOPIC, "payload", statusUpdateEvent) + } + } +} + +func (impl *WorkflowStatusServiceImpl) CheckArgoAppStatusPeriodicallyAndUpdateInDb(getPipelineDeployedBeforeMinutes int, getPipelineDeployedWithinHours int) error { + pipelines, err := impl.pipelineRepository.GetArgoPipelinesHavingLatestTriggerStuckInNonTerminalStatuses(getPipelineDeployedBeforeMinutes, getPipelineDeployedWithinHours) + if err != nil { + impl.logger.Errorw("error in getting pipelines having latest trigger stuck in non terminal statuses", "err", err) + return err + } + impl.logger.Debugw("received stuck argo cd pipelines", "pipelines", pipelines, "number of pipelines", len(pipelines)) + + for _, pipeline := range pipelines { + impl.CheckAndSendArgoPipelineStatusSyncEventIfNeeded(pipeline.Id, 0, 1, false) + } + + installedAppVersions, err := impl.installedAppRepository.GetArgoPipelinesHavingLatestTriggerStuckInNonTerminalStatusesForAppStore(getPipelineDeployedBeforeMinutes, getPipelineDeployedWithinHours) + if err != nil { + impl.logger.Errorw("error in getting installedAppVersions having latest trigger stuck in non terminal statuses", "err", err) + return err + } + impl.logger.Debugw("received stuck argo installed appStore app", "installedAppVersions", installedAppVersions, "number of triggers", len(installedAppVersions)) + + for _, installedAppVersion := range installedAppVersions { + impl.CheckAndSendArgoPipelineStatusSyncEventIfNeeded(0, installedAppVersion.Id, 1, true) + } + return nil +} + +func (impl *WorkflowStatusServiceImpl) CheckArgoPipelineTimelineStatusPeriodicallyAndUpdateInDb(pendingSinceSeconds int, timeForDegradation int) error { + // getting all the progressing status that are stuck since some time after kubectl apply success sync stage + // and are not eligible for CheckArgoAppStatusPeriodicallyAndUpdateInDb + pipelines, err := impl.pipelineRepository.GetArgoPipelinesHavingTriggersStuckInLastPossibleNonTerminalTimelines(pendingSinceSeconds, timeForDegradation) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("err in GetArgoPipelinesHavingTriggersStuckInLastPossibleNonTerminalTimelines", "err", err) + return err + } + impl.logger.Debugw("received argo cd pipelines stuck at kubectl apply synced stage", "pipelines", pipelines) + + installedAppVersions, err := impl.installedAppRepository.GetArgoPipelinesHavingTriggersStuckInLastPossibleNonTerminalTimelinesForAppStore(pendingSinceSeconds, timeForDegradation) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("err in GetArgoPipelinesHavingTriggersStuckInLastPossibleNonTerminalTimelinesForAppStore", "err", err) + return err + } + + impl.logger.Debugw("received argo appStore application stuck at kubectl apply synced stage", "pipelines", installedAppVersions) + for _, pipeline := range pipelines { + impl.CheckAndSendArgoPipelineStatusSyncEventIfNeeded(pipeline.Id, 0, 1, false) + } + + for _, installedAppVersion := range installedAppVersions { + impl.CheckAndSendArgoPipelineStatusSyncEventIfNeeded(0, installedAppVersion.Id, 1, true) + } + return nil +} + +func (impl *WorkflowStatusServiceImpl) syncACDDevtronApps(deployedBeforeMinutes int, pipelineId int) error { + if impl.acdConfig.ArgoCDAutoSyncEnabled { + // don't check for apps if auto sync is enabled + return nil + } + cdWfr, err := impl.cdWorkflowRepository.FindLastStatusByPipelineIdAndRunnerType(pipelineId, bean2.CD_WORKFLOW_TYPE_DEPLOY) + if err != nil { + impl.logger.Errorw("error in getting latest cdWfr by cdPipelineId", "err", err, "pipelineId", pipelineId) + return err + } + if util3.IsTerminalStatus(cdWfr.Status) { + return nil + } + pipelineStatusTimeline, err := impl.pipelineStatusTimelineRepository.FetchLatestTimelineByWfrId(cdWfr.Id) + if err != nil { + impl.logger.Errorw("error in fetching latest pipeline status by cdWfrId", "err", err) + return err + } + if pipelineStatusTimeline.Status == pipelineConfig.TIMELINE_STATUS_ARGOCD_SYNC_INITIATED && time.Since(pipelineStatusTimeline.StatusTime) >= time.Minute*time.Duration(deployedBeforeMinutes) { + acdToken, err := impl.argoUserService.GetLatestDevtronArgoCdUserToken() + if err != nil { + impl.logger.Errorw("error in getting acd token", "err", err) + return err + } + ctx := context.Background() + ctx = context.WithValue(ctx, "token", acdToken) + syncTime := time.Now() + syncErr := impl.argocdClientWrapperService.SyncArgoCDApplicationIfNeededAndRefresh(ctx, cdWfr.CdWorkflow.Pipeline.DeploymentAppName) + if syncErr != nil { + impl.logger.Errorw("error in syncing argoCD app", "err", syncErr) + timelineObject := impl.pipelineStatusTimelineService.GetTimelineDbObjectByTimelineStatusAndTimelineDescription(cdWfr.Id, 0, pipelineConfig.TIMELINE_STATUS_DEPLOYMENT_FAILED, fmt.Sprintf("error occured in syncing argocd application. err: %s", syncErr.Error()), 1, time.Now()) + _ = impl.pipelineStatusTimelineService.SaveTimeline(timelineObject, nil, false) + cdWfr.Status = pipelineConfig.WorkflowFailed + cdWfr.UpdatedBy = 1 + cdWfr.UpdatedOn = time.Now() + cdWfrUpdateErr := impl.cdWorkflowRepository.UpdateWorkFlowRunner(&cdWfr) + if cdWfrUpdateErr != nil { + impl.logger.Errorw("error in updating cd workflow runner as failed in argocd app sync cron", "err", err) + return err + } + return nil + } + timeline := &pipelineConfig.PipelineStatusTimeline{ + CdWorkflowRunnerId: cdWfr.Id, + StatusTime: syncTime, + Status: pipelineConfig.TIMELINE_STATUS_ARGOCD_SYNC_COMPLETED, + StatusDetail: "argocd sync completed", + AuditLog: sql.AuditLog{ + CreatedBy: 1, + CreatedOn: time.Now(), + UpdatedBy: 1, + UpdatedOn: time.Now(), + }, + } + _, err, _ = impl.pipelineStatusTimelineService.SavePipelineStatusTimelineIfNotAlreadyPresent(timeline.CdWorkflowRunnerId, timeline.Status, timeline, false) + } + return nil +} + +func (impl *WorkflowStatusServiceImpl) syncACDHelmApps(deployedBeforeMinutes int, installedAppVersionId int) error { + if impl.acdConfig.ArgoCDAutoSyncEnabled { + // don't check for apps if auto sync is enabled + return nil + } + installedAppVersionHistory, err := impl.installedAppVersionHistoryRepository.GetLatestInstalledAppVersionHistory(installedAppVersionId) + if err != nil { + impl.logger.Errorw("error in getting latest cdWfr by cdPipelineId", "err", err, "installedAppVersionId", installedAppVersionId) + return err + } + if util3.IsTerminalStatus(installedAppVersionHistory.Status) { + return nil + } + installedAppVersionHistoryId := installedAppVersionHistory.Id + pipelineStatusTimeline, err := impl.pipelineStatusTimelineRepository.FetchLatestTimelinesByInstalledAppVersionHistoryId(installedAppVersionHistoryId) + if err != nil { + impl.logger.Errorw("error in fetching latest pipeline status by cdWfrId", "err", err) + return err + } + if pipelineStatusTimeline.Status == pipelineConfig.TIMELINE_STATUS_ARGOCD_SYNC_INITIATED && time.Since(pipelineStatusTimeline.StatusTime) >= time.Minute*time.Duration(deployedBeforeMinutes) { + installedApp, err := impl.installedAppRepository.GetInstalledAppByInstalledAppVersionId(installedAppVersionHistory.InstalledAppVersionId) + if err != nil { + impl.logger.Errorw("error in fetching installed_app by installedAppVersionId", "err", err) + return err + } + appDetails, err := impl.appRepository.FindActiveById(installedApp.AppId) + if err != nil { + impl.logger.Errorw("error in getting appDetails from appId", "err", err) + return err + } + envDetails, err := impl.envRepository.FindById(installedApp.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in fetching environment by envId", "err", err) + } + argoAppName := fmt.Sprintf("%s-%s", appDetails.AppName, envDetails.Name) + acdToken, err := impl.argoUserService.GetLatestDevtronArgoCdUserToken() + if err != nil { + impl.logger.Errorw("error in getting acd token", "err", err) + return err + } + ctx := context.Background() + ctx = context.WithValue(ctx, "token", acdToken) + syncTime := time.Now() + syncErr := impl.argocdClientWrapperService.SyncArgoCDApplicationIfNeededAndRefresh(ctx, argoAppName) + if syncErr != nil { + impl.logger.Errorw("error in syncing argoCD app", "err", syncErr) + timelineObject := impl.pipelineStatusTimelineService.GetTimelineDbObjectByTimelineStatusAndTimelineDescription(0, installedAppVersionHistoryId, pipelineConfig.TIMELINE_STATUS_DEPLOYMENT_FAILED, fmt.Sprintf("error occured in syncing argocd application. err: %s", syncErr.Error()), 1, time.Now()) + _ = impl.pipelineStatusTimelineService.SaveTimeline(timelineObject, nil, false) + installedAppVersionHistory.Status = pipelineConfig.WorkflowFailed + installedAppVersionHistory.UpdatedBy = 1 + installedAppVersionHistory.UpdatedOn = time.Now() + _, installedAppUpdateErr := impl.installedAppVersionHistoryRepository.UpdateInstalledAppVersionHistory(installedAppVersionHistory, nil) + if installedAppUpdateErr != nil { + impl.logger.Errorw("error in updating cd workflow runner as failed in argocd app sync cron", "err", err) + return err + } + return nil + } + timeline := &pipelineConfig.PipelineStatusTimeline{ + InstalledAppVersionHistoryId: installedAppVersionHistoryId, + StatusTime: syncTime, + Status: pipelineConfig.TIMELINE_STATUS_ARGOCD_SYNC_COMPLETED, + StatusDetail: "argocd sync completed", + AuditLog: sql.AuditLog{ + CreatedBy: 1, + CreatedOn: time.Now(), + UpdatedBy: 1, + UpdatedOn: time.Now(), + }, + } + _, err, _ = impl.pipelineStatusTimelineService.SavePipelineStatusTimelineIfNotAlreadyPresent(timeline.CdWorkflowRunnerId, timeline.Status, timeline, false) + } + return nil +} diff --git a/pkg/workflow/status/bean/bean.go b/pkg/workflow/status/bean/bean.go new file mode 100644 index 0000000000..160928a39e --- /dev/null +++ b/pkg/workflow/status/bean/bean.go @@ -0,0 +1,8 @@ +package bean + +type ArgoPipelineStatusSyncEvent struct { + PipelineId int `json:"pipelineId"` + InstalledAppVersionId int `json:"installedAppVersionId"` + UserId int32 `json:"userId"` + IsAppStoreApplication bool `json:"isAppStoreApplication"` +} diff --git a/pkg/workflow/status/wire_workflowStatus.go b/pkg/workflow/status/wire_workflowStatus.go new file mode 100644 index 0000000000..38b37304b3 --- /dev/null +++ b/pkg/workflow/status/wire_workflowStatus.go @@ -0,0 +1,8 @@ +package status + +import "github.com/google/wire" + +var WorkflowStatusWireSet = wire.NewSet( + NewWorkflowStatusServiceImpl, + wire.Bind(new(WorkflowStatusService), new(*WorkflowStatusServiceImpl)), +) diff --git a/pkg/workflow/wire_workflow.go b/pkg/workflow/wire_workflow.go new file mode 100644 index 0000000000..fda1b3fe60 --- /dev/null +++ b/pkg/workflow/wire_workflow.go @@ -0,0 +1,12 @@ +package workflow + +import ( + "github.com/devtron-labs/devtron/pkg/workflow/cd" + "github.com/devtron-labs/devtron/pkg/workflow/status" + "github.com/google/wire" +) + +var WorkflowWireSet = wire.NewSet( + cd.CdWorkflowWireSet, + status.WorkflowStatusWireSet, +) diff --git a/util/GlobalConfig.go b/util/GlobalConfig.go index 1c46f2f25d..4c2d93a32d 100644 --- a/util/GlobalConfig.go +++ b/util/GlobalConfig.go @@ -10,6 +10,7 @@ type GlobalEnvVariables struct { GitOpsRepoPrefix string `env:"GITOPS_REPO_PREFIX" envDefault:""` SkipGitOpsValidation bool `env:"SKIP_GITOPS_VALIDATION" envDefault:"false"` EnableAsyncInstallDevtronChart bool `env:"ENABLE_ASYNC_INSTALL_DEVTRON_CHART" envDefault:"false"` + ExposeCiMetrics bool `env:"EXPOSE_CI_METRICS" envDefault:"false"` } func GetGlobalEnvVariables() (*GlobalEnvVariables, error) { diff --git a/wire_gen.go b/wire_gen.go index 913f9e7bb5..3bcfaeb467 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -49,7 +49,7 @@ import ( "github.com/devtron-labs/devtron/client/argocdServer/application" "github.com/devtron-labs/devtron/client/argocdServer/cluster" "github.com/devtron-labs/devtron/client/argocdServer/connection" - repository11 "github.com/devtron-labs/devtron/client/argocdServer/repository" + repository10 "github.com/devtron-labs/devtron/client/argocdServer/repository" cron2 "github.com/devtron-labs/devtron/client/cron" "github.com/devtron-labs/devtron/client/dashboard" "github.com/devtron-labs/devtron/client/events" @@ -96,6 +96,7 @@ import ( "github.com/devtron-labs/devtron/pkg/auth/sso" "github.com/devtron-labs/devtron/pkg/auth/user" repository4 "github.com/devtron-labs/devtron/pkg/auth/user/repository" + "github.com/devtron-labs/devtron/pkg/build/artifacts" "github.com/devtron-labs/devtron/pkg/bulkAction" "github.com/devtron-labs/devtron/pkg/chart" "github.com/devtron-labs/devtron/pkg/chartRepo" @@ -141,11 +142,11 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/history" - repository8 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" - repository9 "github.com/devtron-labs/devtron/pkg/pipeline/repository" + repository11 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" + repository8 "github.com/devtron-labs/devtron/pkg/pipeline/repository" "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/plugin" - repository10 "github.com/devtron-labs/devtron/pkg/plugin/repository" + repository9 "github.com/devtron-labs/devtron/pkg/plugin/repository" resourceGroup2 "github.com/devtron-labs/devtron/pkg/resourceGroup" "github.com/devtron-labs/devtron/pkg/resourceQualifiers" security2 "github.com/devtron-labs/devtron/pkg/security" @@ -161,6 +162,8 @@ import ( repository6 "github.com/devtron-labs/devtron/pkg/variables/repository" "github.com/devtron-labs/devtron/pkg/webhook/helm" "github.com/devtron-labs/devtron/pkg/workflow/cd" + "github.com/devtron-labs/devtron/pkg/workflow/dag" + status2 "github.com/devtron-labs/devtron/pkg/workflow/status" util2 "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/argo" "github.com/devtron-labs/devtron/util/cron" @@ -378,119 +381,123 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - chartRefRepositoryImpl := chartRepoRepository.NewChartRefRepositoryImpl(db) - utilMergeUtil := util.MergeUtil{ - Logger: sugaredLogger, - } - chartRefServiceImpl := chartRef.NewChartRefServiceImpl(sugaredLogger, chartRefRepositoryImpl, chartTemplateServiceImpl, utilMergeUtil) gitCliUtil := git.NewGitCliUtil(sugaredLogger) gitFactory, err := git.NewGitFactory(sugaredLogger, gitOpsConfigRepositoryImpl, gitCliUtil) if err != nil { return nil, err } gitOperationServiceImpl := git.NewGitOperationServiceImpl(sugaredLogger, gitFactory, gitOpsConfigReadServiceImpl, chartTemplateServiceImpl) + chartRefRepositoryImpl := chartRepoRepository.NewChartRefRepositoryImpl(db) + utilMergeUtil := util.MergeUtil{ + Logger: sugaredLogger, + } + chartRefServiceImpl := chartRef.NewChartRefServiceImpl(sugaredLogger, chartRefRepositoryImpl, chartTemplateServiceImpl, utilMergeUtil) deploymentTemplateServiceImpl := deploymentTemplate.NewDeploymentTemplateServiceImpl(sugaredLogger, chartRefServiceImpl, chartTemplateServiceImpl, chartRepositoryImpl) - appServiceImpl := app2.NewAppService(envConfigOverrideRepositoryImpl, pipelineOverrideRepositoryImpl, mergeUtil, sugaredLogger, pipelineRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, applicationServiceClientImpl, appRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, chartTemplateServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, appStatusServiceImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, globalEnvVariables, scopedVariableCMCSManagerImpl, acdConfig, chartRefServiceImpl, gitOpsConfigReadServiceImpl, gitOperationServiceImpl, deploymentTemplateServiceImpl) + appServiceImpl := app2.NewAppService(envConfigOverrideRepositoryImpl, pipelineOverrideRepositoryImpl, mergeUtil, sugaredLogger, pipelineRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, applicationServiceClientImpl, appRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, chartTemplateServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, appStatusServiceImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, scopedVariableCMCSManagerImpl, acdConfig, gitOpsConfigReadServiceImpl, gitOperationServiceImpl, deploymentTemplateServiceImpl) validate, err := util.IntValidator() if err != nil { return nil, err } - ciCdConfig, err := types.GetCiCdConfig() - if err != nil { - return nil, err - } - globalCMCSRepositoryImpl := repository.NewGlobalCMCSRepositoryImpl(sugaredLogger, db) - globalCMCSServiceImpl := pipeline.NewGlobalCMCSServiceImpl(sugaredLogger, globalCMCSRepositoryImpl) - argoWorkflowExecutorImpl := executors.NewArgoWorkflowExecutorImpl(sugaredLogger) - systemWorkflowExecutorImpl := executors.NewSystemWorkflowExecutorImpl(sugaredLogger, k8sServiceImpl) - k8sCommonServiceImpl := k8s2.NewK8sCommonServiceImpl(sugaredLogger, k8sServiceImpl, clusterServiceImplExtended) - workflowServiceImpl, err := pipeline.NewWorkflowServiceImpl(sugaredLogger, environmentRepositoryImpl, ciCdConfig, appServiceImpl, globalCMCSServiceImpl, argoWorkflowExecutorImpl, k8sServiceImpl, systemWorkflowExecutorImpl, k8sCommonServiceImpl) - if err != nil { - return nil, err - } - materialRepositoryImpl := pipelineConfig.NewMaterialRepositoryImpl(db) - deploymentGroupRepositoryImpl := repository.NewDeploymentGroupRepositoryImpl(sugaredLogger, db) - cvePolicyRepositoryImpl := security.NewPolicyRepositoryImpl(db) - imageScanResultRepositoryImpl := security.NewImageScanResultRepositoryImpl(db, sugaredLogger) appWorkflowRepositoryImpl := appWorkflow.NewAppWorkflowRepositoryImpl(sugaredLogger, db) - prePostCdScriptHistoryRepositoryImpl := repository8.NewPrePostCdScriptHistoryRepositoryImpl(sugaredLogger, db) - configMapHistoryRepositoryImpl := repository8.NewConfigMapHistoryRepositoryImpl(sugaredLogger, db) - configMapHistoryServiceImpl := history.NewConfigMapHistoryServiceImpl(sugaredLogger, configMapHistoryRepositoryImpl, pipelineRepositoryImpl, configMapRepositoryImpl, userServiceImpl, scopedVariableCMCSManagerImpl) - prePostCdScriptHistoryServiceImpl := history.NewPrePostCdScriptHistoryServiceImpl(sugaredLogger, prePostCdScriptHistoryRepositoryImpl, configMapRepositoryImpl, configMapHistoryServiceImpl) - ciTemplateRepositoryImpl := pipelineConfig.NewCiTemplateRepositoryImpl(db, sugaredLogger) - appLabelRepositoryImpl := pipelineConfig.NewAppLabelRepositoryImpl(db) - clientConfig, err := gitSensor.GetConfig() - if err != nil { - return nil, err - } - clientImpl, err := gitSensor.NewGitSensorClient(sugaredLogger, clientConfig) - if err != nil { - return nil, err - } - pipelineStageRepositoryImpl := repository9.NewPipelineStageRepository(sugaredLogger, db) - globalPluginRepositoryImpl := repository10.NewGlobalPluginRepository(sugaredLogger, db) + pipelineStageRepositoryImpl := repository8.NewPipelineStageRepository(sugaredLogger, db) + globalPluginRepositoryImpl := repository9.NewGlobalPluginRepository(sugaredLogger, db) scopedVariableManagerImpl, err := variables.NewScopedVariableManagerImpl(sugaredLogger, scopedVariableServiceImpl, variableEntityMappingServiceImpl, variableSnapshotHistoryServiceImpl, variableTemplateParserImpl) if err != nil { return nil, err } pipelineStageServiceImpl := pipeline.NewPipelineStageService(sugaredLogger, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, pipelineRepositoryImpl, scopedVariableManagerImpl) - globalPluginServiceImpl := plugin.NewGlobalPluginService(sugaredLogger, globalPluginRepositoryImpl, pipelineStageRepositoryImpl) - dockerRegistryIpsConfigRepositoryImpl := repository5.NewDockerRegistryIpsConfigRepositoryImpl(db) - ociRegistryConfigRepositoryImpl := repository5.NewOCIRegistryConfigRepositoryImpl(db) - dockerRegistryConfigImpl := pipeline.NewDockerRegistryConfigImpl(sugaredLogger, helmAppServiceImpl, dockerArtifactStoreRepositoryImpl, dockerRegistryIpsConfigRepositoryImpl, ociRegistryConfigRepositoryImpl) - imageTagRepositoryImpl := repository.NewImageTagRepository(db, sugaredLogger) - customTagServiceImpl := pipeline.NewCustomTagService(sugaredLogger, imageTagRepositoryImpl) - pluginInputVariableParserImpl := pipeline.NewPluginInputVariableParserImpl(sugaredLogger, dockerRegistryConfigImpl, customTagServiceImpl) pipelineConfigListenerServiceImpl := pipeline.NewPipelineConfigListenerServiceImpl(sugaredLogger) - imageDigestPolicyServiceImpl := imageDigestPolicy.NewImageDigestPolicyServiceImpl(sugaredLogger, qualifierMappingServiceImpl, devtronResourceSearchableKeyServiceImpl) - cdWorkflowCommonServiceImpl, err := cd.NewCdWorkflowCommonServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl, pipelineStatusTimelineServiceImpl, pipelineRepositoryImpl) + cdWorkflowCommonServiceImpl, err := cd.NewCdWorkflowCommonServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl, pipelineStatusTimelineServiceImpl, pipelineRepositoryImpl, pipelineStatusTimelineRepositoryImpl) if err != nil { return nil, err } gitOpsManifestPushServiceImpl := app2.NewGitOpsManifestPushServiceImpl(sugaredLogger, pipelineStatusTimelineServiceImpl, pipelineStatusTimelineRepositoryImpl, acdConfig, chartRefServiceImpl, gitOpsConfigReadServiceImpl, gitOperationServiceImpl) argoK8sClientImpl := argocdServer.NewArgoK8sClientImpl(sugaredLogger) - repositoryServiceClientImpl := repository11.NewServiceClientImpl(sugaredLogger, argoCDConnectionManagerImpl) + repositoryServiceClientImpl := repository10.NewServiceClientImpl(sugaredLogger, argoCDConnectionManagerImpl) argoClientWrapperServiceImpl := argocdServer.NewArgoClientWrapperServiceImpl(sugaredLogger, applicationServiceClientImpl, acdConfig, repositoryServiceClientImpl) - imageScanHistoryRepositoryImpl := security.NewImageScanHistoryRepositoryImpl(db, sugaredLogger) - imageScanDeployInfoRepositoryImpl := security.NewImageScanDeployInfoRepositoryImpl(db, sugaredLogger) - manifestPushConfigRepositoryImpl := repository9.NewManifestPushConfigRepository(sugaredLogger, db) - triggerServiceImpl := devtronApps.NewTriggerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, argoK8sClientImpl, acdConfig, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, helmAppServiceImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, globalEnvVariables, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl) - deploymentTemplateHistoryRepositoryImpl := repository8.NewDeploymentTemplateHistoryRepositoryImpl(sugaredLogger, db) + deploymentTemplateHistoryRepositoryImpl := repository11.NewDeploymentTemplateHistoryRepositoryImpl(sugaredLogger, db) appLevelMetricsRepositoryImpl := repository12.NewAppLevelMetricsRepositoryImpl(db, sugaredLogger) envLevelAppMetricsRepositoryImpl := repository12.NewEnvLevelAppMetricsRepositoryImpl(db, sugaredLogger) deployedAppMetricsServiceImpl := deployedAppMetrics.NewDeployedAppMetricsServiceImpl(sugaredLogger, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, chartRefServiceImpl) deploymentTemplateHistoryServiceImpl := history.NewDeploymentTemplateHistoryServiceImpl(sugaredLogger, deploymentTemplateHistoryRepositoryImpl, pipelineRepositoryImpl, chartRepositoryImpl, userServiceImpl, cdWorkflowRepositoryImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) - pipelineStrategyHistoryRepositoryImpl := repository8.NewPipelineStrategyHistoryRepositoryImpl(sugaredLogger, db) + pipelineStrategyHistoryRepositoryImpl := repository11.NewPipelineStrategyHistoryRepositoryImpl(sugaredLogger, db) pipelineStrategyHistoryServiceImpl := history.NewPipelineStrategyHistoryServiceImpl(sugaredLogger, pipelineStrategyHistoryRepositoryImpl, userServiceImpl) + configMapHistoryRepositoryImpl := repository11.NewConfigMapHistoryRepositoryImpl(sugaredLogger, db) + configMapHistoryServiceImpl := history.NewConfigMapHistoryServiceImpl(sugaredLogger, configMapHistoryRepositoryImpl, pipelineRepositoryImpl, configMapRepositoryImpl, userServiceImpl, scopedVariableCMCSManagerImpl) deployedConfigurationHistoryServiceImpl := history.NewDeployedConfigurationHistoryServiceImpl(sugaredLogger, userServiceImpl, deploymentTemplateHistoryServiceImpl, pipelineStrategyHistoryServiceImpl, configMapHistoryServiceImpl, cdWorkflowRepositoryImpl, scopedVariableCMCSManagerImpl) + dockerRegistryIpsConfigRepositoryImpl := repository5.NewDockerRegistryIpsConfigRepositoryImpl(db) ciTemplateOverrideRepositoryImpl := pipelineConfig.NewCiTemplateOverrideRepositoryImpl(db, sugaredLogger) dockerRegistryIpsConfigServiceImpl := dockerRegistry.NewDockerRegistryIpsConfigServiceImpl(sugaredLogger, dockerRegistryIpsConfigRepositoryImpl, k8sServiceImpl, clusterServiceImplExtended, ciPipelineRepositoryImpl, dockerArtifactStoreRepositoryImpl, ciTemplateOverrideRepositoryImpl) + k8sCommonServiceImpl := k8s2.NewK8sCommonServiceImpl(sugaredLogger, k8sServiceImpl, clusterServiceImplExtended) + imageDigestPolicyServiceImpl := imageDigestPolicy.NewImageDigestPolicyServiceImpl(sugaredLogger, qualifierMappingServiceImpl, devtronResourceSearchableKeyServiceImpl) + appLabelRepositoryImpl := pipelineConfig.NewAppLabelRepositoryImpl(db) genericNoteRepositoryImpl := repository13.NewGenericNoteRepositoryImpl(db) genericNoteHistoryRepositoryImpl := repository13.NewGenericNoteHistoryRepositoryImpl(db) genericNoteHistoryServiceImpl := genericNotes.NewGenericNoteHistoryServiceImpl(genericNoteHistoryRepositoryImpl, sugaredLogger) genericNoteServiceImpl := genericNotes.NewGenericNoteServiceImpl(genericNoteRepositoryImpl, genericNoteHistoryServiceImpl, userRepositoryImpl, sugaredLogger) + materialRepositoryImpl := pipelineConfig.NewMaterialRepositoryImpl(db) appCrudOperationServiceImpl := app2.NewAppCrudOperationServiceImpl(appLabelRepositoryImpl, sugaredLogger, appRepositoryImpl, userRepositoryImpl, installedAppRepositoryImpl, genericNoteServiceImpl, materialRepositoryImpl) pipelineConfigRepositoryImpl := chartConfig.NewPipelineConfigRepository(db) - manifestCreationServiceImpl := manifest.NewManifestCreationServiceImpl(sugaredLogger, dockerRegistryIpsConfigServiceImpl, chartRefServiceImpl, scopedVariableCMCSManagerImpl, k8sCommonServiceImpl, deployedAppMetricsServiceImpl, imageDigestPolicyServiceImpl, mergeUtil, appCrudOperationServiceImpl, applicationServiceClientImpl, configMapHistoryRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, envConfigOverrideRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineOverrideRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, pipelineConfigRepositoryImpl, deploymentTemplateHistoryRepositoryImpl) - workflowEventPublishServiceImpl, err := out.NewWorkflowEventPublishServiceImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowCommonServiceImpl, deployedConfigurationHistoryServiceImpl, manifestCreationServiceImpl, pipelineStatusTimelineServiceImpl, cdWorkflowRepositoryImpl, pipelineRepositoryImpl) + manifestCreationServiceImpl := manifest.NewManifestCreationServiceImpl(sugaredLogger, dockerRegistryIpsConfigServiceImpl, chartRefServiceImpl, scopedVariableCMCSManagerImpl, k8sCommonServiceImpl, deployedAppMetricsServiceImpl, imageDigestPolicyServiceImpl, mergeUtil, appCrudOperationServiceImpl, deploymentTemplateServiceImpl, applicationServiceClientImpl, configMapHistoryRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, envConfigOverrideRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineOverrideRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, pipelineConfigRepositoryImpl, deploymentTemplateHistoryRepositoryImpl) + deploymentGroupRepositoryImpl := repository.NewDeploymentGroupRepositoryImpl(sugaredLogger, db) + workflowEventPublishServiceImpl, err := out.NewWorkflowEventPublishServiceImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowCommonServiceImpl, deployedConfigurationHistoryServiceImpl, manifestCreationServiceImpl, pipelineStatusTimelineServiceImpl, cdWorkflowRepositoryImpl, pipelineRepositoryImpl, deploymentGroupRepositoryImpl) + if err != nil { + return nil, err + } + globalPluginServiceImpl := plugin.NewGlobalPluginService(sugaredLogger, globalPluginRepositoryImpl, pipelineStageRepositoryImpl) + imageTagRepositoryImpl := repository.NewImageTagRepository(db, sugaredLogger) + customTagServiceImpl := pipeline.NewCustomTagService(sugaredLogger, imageTagRepositoryImpl) + ociRegistryConfigRepositoryImpl := repository5.NewOCIRegistryConfigRepositoryImpl(db) + dockerRegistryConfigImpl := pipeline.NewDockerRegistryConfigImpl(sugaredLogger, helmAppServiceImpl, dockerArtifactStoreRepositoryImpl, dockerRegistryIpsConfigRepositoryImpl, ociRegistryConfigRepositoryImpl) + pluginInputVariableParserImpl := pipeline.NewPluginInputVariableParserImpl(sugaredLogger, dockerRegistryConfigImpl, customTagServiceImpl) + prePostCdScriptHistoryRepositoryImpl := repository11.NewPrePostCdScriptHistoryRepositoryImpl(sugaredLogger, db) + prePostCdScriptHistoryServiceImpl := history.NewPrePostCdScriptHistoryServiceImpl(sugaredLogger, prePostCdScriptHistoryRepositoryImpl, configMapRepositoryImpl, configMapHistoryServiceImpl) + ciCdConfig, err := types.GetCiCdConfig() + if err != nil { + return nil, err + } + globalCMCSRepositoryImpl := repository.NewGlobalCMCSRepositoryImpl(sugaredLogger, db) + globalCMCSServiceImpl := pipeline.NewGlobalCMCSServiceImpl(sugaredLogger, globalCMCSRepositoryImpl) + argoWorkflowExecutorImpl := executors.NewArgoWorkflowExecutorImpl(sugaredLogger) + systemWorkflowExecutorImpl := executors.NewSystemWorkflowExecutorImpl(sugaredLogger, k8sServiceImpl) + workflowServiceImpl, err := pipeline.NewWorkflowServiceImpl(sugaredLogger, environmentRepositoryImpl, ciCdConfig, appServiceImpl, globalCMCSServiceImpl, argoWorkflowExecutorImpl, k8sServiceImpl, systemWorkflowExecutorImpl, k8sCommonServiceImpl) if err != nil { return nil, err } - workflowDagExecutorImpl := pipeline.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, pubSubClientServiceImpl, workflowServiceImpl, ciArtifactRepositoryImpl, ciPipelineRepositoryImpl, materialRepositoryImpl, pipelineOverrideRepositoryImpl, userServiceImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, enforcerUtilImpl, eventSimpleFactoryImpl, eventRESTClientImpl, cvePolicyRepositoryImpl, imageScanResultRepositoryImpl, appWorkflowRepositoryImpl, prePostCdScriptHistoryServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineServiceImpl, ciTemplateRepositoryImpl, ciWorkflowRepositoryImpl, appLabelRepositoryImpl, clientImpl, pipelineStageServiceImpl, k8sCommonServiceImpl, globalPluginServiceImpl, pluginInputVariableParserImpl, scopedVariableCMCSManagerImpl, helmAppServiceImpl, pipelineConfigListenerServiceImpl, customTagServiceImpl, imageDigestPolicyServiceImpl, cdWorkflowCommonServiceImpl, triggerServiceImpl, deployedConfigurationHistoryServiceImpl, workflowEventPublishServiceImpl, manifestCreationServiceImpl, deploymentTemplateServiceImpl) + clientConfig, err := gitSensor.GetConfig() + if err != nil { + return nil, err + } + clientImpl, err := gitSensor.NewGitSensorClient(sugaredLogger, clientConfig) + if err != nil { + return nil, err + } + imageScanResultRepositoryImpl := security.NewImageScanResultRepositoryImpl(db, sugaredLogger) + cvePolicyRepositoryImpl := security.NewPolicyRepositoryImpl(db) + imageScanHistoryRepositoryImpl := security.NewImageScanHistoryRepositoryImpl(db, sugaredLogger) + imageScanDeployInfoRepositoryImpl := security.NewImageScanDeployInfoRepositoryImpl(db, sugaredLogger) + manifestPushConfigRepositoryImpl := repository8.NewManifestPushConfigRepository(sugaredLogger, db) + ciTemplateRepositoryImpl := pipelineConfig.NewCiTemplateRepositoryImpl(db, sugaredLogger) + triggerServiceImpl, err := devtronApps.NewTriggerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, argoK8sClientImpl, acdConfig, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, workflowEventPublishServiceImpl, manifestCreationServiceImpl, deployedConfigurationHistoryServiceImpl, argoUserServiceImpl, pipelineStageServiceImpl, globalPluginServiceImpl, customTagServiceImpl, pluginInputVariableParserImpl, prePostCdScriptHistoryServiceImpl, scopedVariableCMCSManagerImpl, workflowServiceImpl, imageDigestPolicyServiceImpl, userServiceImpl, clientImpl, helmAppServiceImpl, enforcerUtilImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, globalEnvVariables, imageScanResultRepositoryImpl, cvePolicyRepositoryImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, cdWorkflowRepositoryImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciTemplateRepositoryImpl, materialRepositoryImpl, appLabelRepositoryImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl) + if err != nil { + return nil, err + } + commonArtifactServiceImpl := artifacts.NewCommonArtifactServiceImpl(sugaredLogger, ciArtifactRepositoryImpl) + workflowDagExecutorImpl := dag.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, pubSubClientServiceImpl, ciArtifactRepositoryImpl, enforcerUtilImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, ciWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, helmAppServiceImpl, pipelineConfigListenerServiceImpl, cdWorkflowCommonServiceImpl, triggerServiceImpl, manifestCreationServiceImpl, commonArtifactServiceImpl) deploymentGroupAppRepositoryImpl := repository.NewDeploymentGroupAppRepositoryImpl(sugaredLogger, db) deploymentGroupServiceImpl := deploymentGroup.NewDeploymentGroupServiceImpl(appRepositoryImpl, sugaredLogger, pipelineRepositoryImpl, ciPipelineRepositoryImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, deploymentGroupAppRepositoryImpl, ciArtifactRepositoryImpl, appWorkflowRepositoryImpl, workflowEventPublishServiceImpl) deploymentConfigServiceImpl := pipeline.NewDeploymentConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, pipelineRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, configMapHistoryServiceImpl, scopedVariableCMCSManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) - deployedAppServiceImpl := deployedApp.NewDeployedAppServiceImpl(sugaredLogger, k8sCommonServiceImpl, environmentRepositoryImpl) - pipelineTriggerRestHandlerImpl := restHandler.NewPipelineRestHandler(appServiceImpl, userServiceImpl, validate, enforcerImpl, teamServiceImpl, sugaredLogger, enforcerUtilImpl, workflowDagExecutorImpl, deploymentGroupServiceImpl, argoUserServiceImpl, deploymentConfigServiceImpl, deployedAppServiceImpl) + deployedAppServiceImpl := deployedApp.NewDeployedAppServiceImpl(sugaredLogger, k8sCommonServiceImpl, triggerServiceImpl, environmentRepositoryImpl, pipelineRepositoryImpl, cdWorkflowRepositoryImpl) + pipelineTriggerRestHandlerImpl := restHandler.NewPipelineRestHandler(appServiceImpl, userServiceImpl, validate, enforcerImpl, teamServiceImpl, sugaredLogger, enforcerUtilImpl, workflowDagExecutorImpl, deploymentGroupServiceImpl, argoUserServiceImpl, deploymentConfigServiceImpl, deployedAppServiceImpl, triggerServiceImpl, workflowEventPublishServiceImpl) sseSSE := sse.NewSSE() pipelineTriggerRouterImpl := router.NewPipelineTriggerRouter(pipelineTriggerRestHandlerImpl, sseSSE) appListingRepositoryQueryBuilder := helper.NewAppListingRepositoryQueryBuilder(sugaredLogger) appListingRepositoryImpl := repository.NewAppListingRepositoryImpl(sugaredLogger, db, appListingRepositoryQueryBuilder, environmentRepositoryImpl) - prePostCiScriptHistoryRepositoryImpl := repository8.NewPrePostCiScriptHistoryRepositoryImpl(sugaredLogger, db) + prePostCiScriptHistoryRepositoryImpl := repository11.NewPrePostCiScriptHistoryRepositoryImpl(sugaredLogger, db) prePostCiScriptHistoryServiceImpl := history.NewPrePostCiScriptHistoryServiceImpl(sugaredLogger, prePostCiScriptHistoryRepositoryImpl) - gitMaterialHistoryRepositoryImpl := repository8.NewGitMaterialHistoryRepositoyImpl(db) + gitMaterialHistoryRepositoryImpl := repository11.NewGitMaterialHistoryRepositoyImpl(db) gitMaterialHistoryServiceImpl := history.NewGitMaterialHistoryServiceImpl(gitMaterialHistoryRepositoryImpl, sugaredLogger) - ciPipelineHistoryRepositoryImpl := repository8.NewCiPipelineHistoryRepositoryImpl(db, sugaredLogger) + ciPipelineHistoryRepositoryImpl := repository11.NewCiPipelineHistoryRepositoryImpl(db, sugaredLogger) ciPipelineHistoryServiceImpl := history.NewCiPipelineHistoryServiceImpl(ciPipelineHistoryRepositoryImpl, sugaredLogger, ciPipelineRepositoryImpl) ciBuildConfigRepositoryImpl := pipelineConfig.NewCiBuildConfigRepositoryImpl(db, sugaredLogger) ciBuildConfigServiceImpl := pipeline.NewCiBuildConfigServiceImpl(sugaredLogger, ciBuildConfigRepositoryImpl) @@ -502,7 +509,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - ciTemplateHistoryRepositoryImpl := repository8.NewCiTemplateHistoryRepositoryImpl(db, sugaredLogger) + ciTemplateHistoryRepositoryImpl := repository11.NewCiTemplateHistoryRepositoryImpl(db, sugaredLogger) ciTemplateHistoryServiceImpl := history.NewCiTemplateHistoryServiceImpl(ciTemplateHistoryRepositoryImpl, sugaredLogger) resourceGroupRepositoryImpl := resourceGroup.NewResourceGroupRepositoryImpl(db) resourceGroupMappingRepositoryImpl := resourceGroup.NewResourceGroupMappingRepositoryImpl(db) @@ -535,12 +542,11 @@ func InitializeApp() (*App, error) { blobStorageConfigServiceImpl := pipeline.NewBlobStorageConfigServiceImpl(sugaredLogger, k8sServiceImpl, ciCdConfig) ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, workflowServiceImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, k8sServiceImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl, environmentServiceImpl) gitRegistryConfigImpl := pipeline.NewGitRegistryConfigImpl(sugaredLogger, gitProviderRepositoryImpl, clientImpl) + cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sServiceImpl, workflowServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl, customTagServiceImpl) + appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, appRepositoryImpl, userAuthServiceImpl) appListingViewBuilderImpl := app2.NewAppListingViewBuilderImpl(sugaredLogger) linkoutsRepositoryImpl := repository.NewLinkoutsRepositoryImpl(sugaredLogger, db) appListingServiceImpl := app2.NewAppListingServiceImpl(sugaredLogger, appListingRepositoryImpl, applicationServiceClientImpl, appRepositoryImpl, appListingViewBuilderImpl, pipelineRepositoryImpl, linkoutsRepositoryImpl, cdWorkflowRepositoryImpl, pipelineOverrideRepositoryImpl, environmentRepositoryImpl, argoUserServiceImpl, envConfigOverrideRepositoryImpl, chartRepositoryImpl, ciPipelineRepositoryImpl, dockerRegistryIpsConfigServiceImpl, userRepositoryImpl, deployedAppMetricsServiceImpl) - deploymentEventHandlerImpl := app2.NewDeploymentEventHandlerImpl(sugaredLogger, appListingServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl) - cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, helmAppServiceImpl, pipelineOverrideRepositoryImpl, workflowDagExecutorImpl, appListingServiceImpl, appListingRepositoryImpl, pipelineStatusTimelineRepositoryImpl, applicationServiceClientImpl, argoUserServiceImpl, deploymentEventHandlerImpl, eventRESTClientImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceImpl, appStatusServiceImpl, enforcerUtilImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, appRepositoryImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sServiceImpl, workflowServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl, customTagServiceImpl, argoClientWrapperServiceImpl, appServiceConfig, acdConfig) - appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, appRepositoryImpl, userAuthServiceImpl) appCloneServiceImpl := appClone.NewAppCloneServiceImpl(sugaredLogger, pipelineBuilderImpl, chartServiceImpl, configMapServiceImpl, appWorkflowServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, pipelineStageServiceImpl, ciTemplateServiceImpl, appRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, ciPipelineConfigServiceImpl, gitOpsConfigReadServiceImpl) deploymentTemplateRepositoryImpl := repository.NewDeploymentTemplateRepositoryImpl(db, sugaredLogger) generateManifestDeploymentTemplateServiceImpl := generateManifest.NewDeploymentTemplateServiceImpl(sugaredLogger, chartServiceImpl, appListingServiceImpl, appListingRepositoryImpl, deploymentTemplateRepositoryImpl, helmAppServiceImpl, chartRepositoryImpl, chartTemplateServiceImpl, helmAppClientImpl, k8sServiceImpl, propertiesConfigServiceImpl, deploymentTemplateHistoryServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, chartRefServiceImpl) @@ -584,7 +590,11 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - cdApplicationStatusUpdateHandlerImpl := cron2.NewCdApplicationStatusUpdateHandlerImpl(sugaredLogger, appServiceImpl, workflowDagExecutorImpl, installedAppServiceImpl, cdHandlerImpl, appServiceConfig, pubSubClientServiceImpl, pipelineStatusTimelineRepositoryImpl, eventRESTClientImpl, appListingRepositoryImpl, cdWorkflowRepositoryImpl, pipelineRepositoryImpl, installedAppVersionHistoryRepositoryImpl, installedAppRepositoryImpl, cronLoggerImpl) + workflowStatusServiceImpl, err := status2.NewWorkflowStatusServiceImpl(sugaredLogger, workflowDagExecutorImpl, pipelineStatusTimelineServiceImpl, appServiceImpl, appStatusServiceImpl, acdConfig, appServiceConfig, argoUserServiceImpl, pipelineStatusSyncDetailServiceImpl, argoClientWrapperServiceImpl, cdWorkflowRepositoryImpl, pipelineOverrideRepositoryImpl, installedAppVersionHistoryRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, installedAppRepositoryImpl, pipelineStatusTimelineRepositoryImpl, pipelineRepositoryImpl, applicationServiceClientImpl, eventRESTClientImpl) + if err != nil { + return nil, err + } + cdApplicationStatusUpdateHandlerImpl := cron2.NewCdApplicationStatusUpdateHandlerImpl(sugaredLogger, appServiceImpl, workflowDagExecutorImpl, installedAppServiceImpl, appServiceConfig, pubSubClientServiceImpl, pipelineStatusTimelineRepositoryImpl, eventRESTClientImpl, appListingRepositoryImpl, cdWorkflowRepositoryImpl, pipelineRepositoryImpl, installedAppVersionHistoryRepositoryImpl, installedAppRepositoryImpl, cronLoggerImpl, cdWorkflowCommonServiceImpl, workflowStatusServiceImpl) appListingRestHandlerImpl := restHandler.NewAppListingRestHandlerImpl(applicationServiceClientImpl, appListingServiceImpl, teamServiceImpl, enforcerImpl, pipelineBuilderImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, userServiceImpl, helmAppClientImpl, clusterServiceImplExtended, helmAppServiceImpl, argoUserServiceImpl, k8sCommonServiceImpl, installedAppServiceImpl, cdApplicationStatusUpdateHandlerImpl, pipelineRepositoryImpl, appStatusServiceImpl, installedAppRepositoryImpl, environmentServiceImpl, genericNoteServiceImpl, k8sApplicationServiceImpl, generateManifestDeploymentTemplateServiceImpl) appListingRouterImpl := router.NewAppListingRouterImpl(appListingRestHandlerImpl) chartRepositoryServiceImpl := chartRepo.NewChartRepositoryServiceImpl(sugaredLogger, chartRepoRepositoryImpl, k8sServiceImpl, clusterServiceImplExtended, acdAuthConfig, httpClient, serverEnvConfigServerEnvConfig) @@ -599,13 +609,7 @@ func InitializeApp() (*App, error) { gitWebhookRepositoryImpl := repository.NewGitWebhookRepositoryImpl(db) gitWebhookServiceImpl := git2.NewGitWebhookServiceImpl(sugaredLogger, ciHandlerImpl, gitWebhookRepositoryImpl) gitWebhookRestHandlerImpl := restHandler.NewGitWebhookRestHandlerImpl(sugaredLogger, gitWebhookServiceImpl) - webhookServiceImpl := pipeline.NewWebhookServiceImpl(ciArtifactRepositoryImpl, sugaredLogger, ciPipelineRepositoryImpl, appServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciWorkflowRepositoryImpl, workflowDagExecutorImpl, ciHandlerImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, customTagServiceImpl) - ciEventConfig, err := pubsub.GetCiEventConfig() - if err != nil { - return nil, err - } - ciEventHandlerImpl := pubsub.NewCiEventHandlerImpl(sugaredLogger, pubSubClientServiceImpl, webhookServiceImpl, validate, ciEventConfig) - externalCiRestHandlerImpl := restHandler.NewExternalCiRestHandlerImpl(sugaredLogger, webhookServiceImpl, ciEventHandlerImpl, validate, userServiceImpl, enforcerImpl, enforcerUtilImpl) + externalCiRestHandlerImpl := restHandler.NewExternalCiRestHandlerImpl(sugaredLogger, validate, userServiceImpl, enforcerImpl, workflowDagExecutorImpl) pubSubClientRestHandlerImpl := restHandler.NewPubSubClientRestHandlerImpl(pubSubClientServiceImpl, sugaredLogger, ciCdConfig) webhookRouterImpl := router.NewWebhookRouterImpl(gitWebhookRestHandlerImpl, pipelineConfigRestHandlerImpl, externalCiRestHandlerImpl, pubSubClientRestHandlerImpl) userAuthHandlerImpl := user2.NewUserAuthHandlerImpl(userAuthServiceImpl, validate, sugaredLogger, enforcerImpl) @@ -655,7 +659,7 @@ func InitializeApp() (*App, error) { teamRestHandlerImpl := team2.NewTeamRestHandlerImpl(sugaredLogger, teamServiceImpl, userServiceImpl, enforcerImpl, validate, userAuthServiceImpl, deleteServiceExtendedImpl) teamRouterImpl := team2.NewTeamRouterImpl(teamRestHandlerImpl) gitWebhookHandlerImpl := pubsub.NewGitWebhookHandler(sugaredLogger, pubSubClientServiceImpl, gitWebhookServiceImpl) - applicationStatusHandlerImpl := pubsub.NewApplicationStatusHandlerImpl(sugaredLogger, pubSubClientServiceImpl, appServiceImpl, workflowDagExecutorImpl, installedAppServiceImpl, appStoreDeploymentServiceImpl, pipelineBuilderImpl, pipelineRepositoryImpl, installedAppRepositoryImpl, gitOpsConfigReadServiceImpl) + applicationStatusHandlerImpl := pubsub.NewApplicationStatusHandlerImpl(sugaredLogger, pubSubClientServiceImpl, appServiceImpl, workflowDagExecutorImpl, installedAppServiceImpl, appStoreDeploymentServiceImpl, pipelineBuilderImpl, pipelineRepositoryImpl, installedAppRepositoryImpl, gitOpsConfigReadServiceImpl, cdWorkflowCommonServiceImpl) roleGroupServiceImpl := user.NewRoleGroupServiceImpl(userAuthRepositoryImpl, sugaredLogger, userRepositoryImpl, roleGroupRepositoryImpl, userCommonServiceImpl) userRestHandlerImpl := user2.NewUserRestHandlerImpl(userServiceImpl, validate, sugaredLogger, enforcerImpl, roleGroupServiceImpl, userCommonServiceImpl) userRouterImpl := user2.NewUserRouterImpl(userRestHandlerImpl) @@ -745,7 +749,7 @@ func InitializeApp() (*App, error) { telemetryRestHandlerImpl := restHandler.NewTelemetryRestHandlerImpl(sugaredLogger, telemetryEventClientImplExtended, enforcerImpl, userServiceImpl) telemetryRouterImpl := router.NewTelemetryRouterImpl(sugaredLogger, telemetryRestHandlerImpl) bulkUpdateRepositoryImpl := bulkUpdate.NewBulkUpdateRepository(db, sugaredLogger) - bulkUpdateServiceImpl, err := bulkAction.NewBulkUpdateServiceImpl(bulkUpdateRepositoryImpl, chartRepositoryImpl, sugaredLogger, environmentRepositoryImpl, pipelineRepositoryImpl, appRepositoryImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, workflowDagExecutorImpl, pipelineBuilderImpl, enforcerUtilImpl, ciHandlerImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, appWorkflowServiceImpl, pubSubClientServiceImpl, argoUserServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) + bulkUpdateServiceImpl, err := bulkAction.NewBulkUpdateServiceImpl(bulkUpdateRepositoryImpl, sugaredLogger, environmentRepositoryImpl, pipelineRepositoryImpl, appRepositoryImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, workflowDagExecutorImpl, pipelineBuilderImpl, enforcerUtilImpl, ciHandlerImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, appWorkflowServiceImpl, pubSubClientServiceImpl, argoUserServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, triggerServiceImpl, deployedAppServiceImpl, cdWorkflowCommonServiceImpl) if err != nil { return nil, err } @@ -838,11 +842,12 @@ func InitializeApp() (*App, error) { return nil, err } proxyRouterImpl := proxy.NewProxyRouterImpl(sugaredLogger, proxyConfig, enforcerImpl) - muxRouter := router.NewMuxRouter(sugaredLogger, pipelineTriggerRouterImpl, pipelineConfigRouterImpl, appListingRouterImpl, environmentRouterImpl, clusterRouterImpl, webhookRouterImpl, userAuthRouterImpl, applicationRouterImpl, cdRouterImpl, gitProviderRouterImpl, gitHostRouterImpl, dockerRegRouterImpl, notificationRouterImpl, teamRouterImpl, gitWebhookHandlerImpl, applicationStatusHandlerImpl, ciEventHandlerImpl, pubSubClientServiceImpl, userRouterImpl, chartRefRouterImpl, configMapRouterImpl, appStoreRouterImpl, chartRepositoryRouterImpl, releaseMetricsRouterImpl, deploymentGroupRouterImpl, batchOperationRouterImpl, chartGroupRouterImpl, imageScanRouterImpl, policyRouterImpl, gitOpsConfigRouterImpl, dashboardRouterImpl, attributesRouterImpl, userAttributesRouterImpl, commonRouterImpl, grafanaRouterImpl, ssoLoginRouterImpl, telemetryRouterImpl, telemetryEventClientImplExtended, bulkUpdateRouterImpl, webhookListenerRouterImpl, appRouterImpl, coreAppRouterImpl, helmAppRouterImpl, k8sApplicationRouterImpl, pProfRouterImpl, deploymentConfigRouterImpl, dashboardTelemetryRouterImpl, commonDeploymentRouterImpl, externalLinkRouterImpl, globalPluginRouterImpl, moduleRouterImpl, serverRouterImpl, apiTokenRouterImpl, cdApplicationStatusUpdateHandlerImpl, k8sCapacityRouterImpl, webhookHelmRouterImpl, globalCMCSRouterImpl, userTerminalAccessRouterImpl, jobRouterImpl, ciStatusUpdateCronImpl, resourceGroupingRouterImpl, rbacRoleRouterImpl, scopedVariableRouterImpl, ciTriggerCronImpl, proxyRouterImpl) + muxRouter := router.NewMuxRouter(sugaredLogger, pipelineTriggerRouterImpl, pipelineConfigRouterImpl, appListingRouterImpl, environmentRouterImpl, clusterRouterImpl, webhookRouterImpl, userAuthRouterImpl, applicationRouterImpl, cdRouterImpl, gitProviderRouterImpl, gitHostRouterImpl, dockerRegRouterImpl, notificationRouterImpl, teamRouterImpl, gitWebhookHandlerImpl, applicationStatusHandlerImpl, pubSubClientServiceImpl, userRouterImpl, chartRefRouterImpl, configMapRouterImpl, appStoreRouterImpl, chartRepositoryRouterImpl, releaseMetricsRouterImpl, deploymentGroupRouterImpl, batchOperationRouterImpl, chartGroupRouterImpl, imageScanRouterImpl, policyRouterImpl, gitOpsConfigRouterImpl, dashboardRouterImpl, attributesRouterImpl, userAttributesRouterImpl, commonRouterImpl, grafanaRouterImpl, ssoLoginRouterImpl, telemetryRouterImpl, telemetryEventClientImplExtended, bulkUpdateRouterImpl, webhookListenerRouterImpl, appRouterImpl, coreAppRouterImpl, helmAppRouterImpl, k8sApplicationRouterImpl, pProfRouterImpl, deploymentConfigRouterImpl, dashboardTelemetryRouterImpl, commonDeploymentRouterImpl, externalLinkRouterImpl, globalPluginRouterImpl, moduleRouterImpl, serverRouterImpl, apiTokenRouterImpl, cdApplicationStatusUpdateHandlerImpl, k8sCapacityRouterImpl, webhookHelmRouterImpl, globalCMCSRouterImpl, userTerminalAccessRouterImpl, jobRouterImpl, ciStatusUpdateCronImpl, resourceGroupingRouterImpl, rbacRoleRouterImpl, scopedVariableRouterImpl, ciTriggerCronImpl, proxyRouterImpl) loggingMiddlewareImpl := util4.NewLoggingMiddlewareImpl(userServiceImpl) cdWorkflowServiceImpl := cd.NewCdWorkflowServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) cdWorkflowRunnerServiceImpl := cd.NewCdWorkflowRunnerServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) - workflowEventProcessorImpl, err := in.NewWorkflowEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowServiceImpl, cdWorkflowRunnerServiceImpl, workflowDagExecutorImpl, argoUserServiceImpl, ciHandlerImpl, cdHandlerImpl, eventSimpleFactoryImpl, eventRESTClientImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, cdWorkflowRepositoryImpl) + webhookServiceImpl := pipeline.NewWebhookServiceImpl(ciArtifactRepositoryImpl, sugaredLogger, ciPipelineRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowCommonServiceImpl) + workflowEventProcessorImpl, err := in.NewWorkflowEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowServiceImpl, cdWorkflowRunnerServiceImpl, workflowDagExecutorImpl, argoUserServiceImpl, ciHandlerImpl, cdHandlerImpl, eventSimpleFactoryImpl, eventRESTClientImpl, triggerServiceImpl, deployedAppServiceImpl, webhookServiceImpl, validate, globalEnvVariables, cdWorkflowCommonServiceImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, cdWorkflowRepositoryImpl) if err != nil { return nil, err } From 7e0dcf5a4f6d14450eb95d123c4c409f1d687039 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 6 Feb 2024 16:03:25 +0530 Subject: [PATCH 59/64] replaced slices -> k8s.io/utils/strings/slices import --- pkg/workflow/cd/CdWorkflowCommonService.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/workflow/cd/CdWorkflowCommonService.go b/pkg/workflow/cd/CdWorkflowCommonService.go index 0fda3767f0..65a91d236d 100644 --- a/pkg/workflow/cd/CdWorkflowCommonService.go +++ b/pkg/workflow/cd/CdWorkflowCommonService.go @@ -16,7 +16,7 @@ import ( util4 "github.com/devtron-labs/devtron/util" "github.com/go-pg/pg" "go.uber.org/zap" - "slices" + "k8s.io/utils/strings/slices" "time" ) From 0aa56d06572a581a5d7192a4de77a9f05b0b0cea Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 6 Feb 2024 16:13:10 +0530 Subject: [PATCH 60/64] replaced slices -> k8s.io/utils/strings/slices import --- pkg/workflow/status/WorkflowStatusService.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/workflow/status/WorkflowStatusService.go b/pkg/workflow/status/WorkflowStatusService.go index 05534b5d9d..82112ad0f0 100644 --- a/pkg/workflow/status/WorkflowStatusService.go +++ b/pkg/workflow/status/WorkflowStatusService.go @@ -27,7 +27,7 @@ import ( "github.com/devtron-labs/devtron/util/argo" "github.com/go-pg/pg" "go.uber.org/zap" - "slices" + "k8s.io/utils/strings/slices" "time" ) From ee52ac40bfd3cf4343083a3c3b1bcade4c19fc0d Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Thu, 8 Feb 2024 17:18:14 +0530 Subject: [PATCH 61/64] fix prod bug --- internal/sql/repository/DeploymentTemplateRepository.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sql/repository/DeploymentTemplateRepository.go b/internal/sql/repository/DeploymentTemplateRepository.go index c35df61721..07946e5f51 100644 --- a/internal/sql/repository/DeploymentTemplateRepository.go +++ b/internal/sql/repository/DeploymentTemplateRepository.go @@ -53,7 +53,7 @@ func (impl DeploymentTemplateRepositoryImpl) FetchDeploymentHistoryWithChartRefs limit := 15 query := "select p.id as pipeline_id, dth.id as deployment_template_history_id," + - " wfr.finished_on, wfr.status, ceco.chart_id, c.chart_version FROM cd_workflow_runner wfr" + + " wfr.finished_on, wfr.status, c.chart_ref_id, c.chart_version FROM cd_workflow_runner wfr" + " JOIN cd_workflow wf ON wf.id = wfr.cd_workflow_id JOIN pipeline p ON p.id = wf.pipeline_id" + " JOIN deployment_template_history dth ON dth.deployed_on = wfr.started_on " + "JOIN pipeline_config_override pco ON pco.cd_workflow_id = wf.id " + From b3df6bf29f393c04e2104a9e0f6b4c59c8ce6cb9 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Fri, 9 Feb 2024 12:26:53 +0530 Subject: [PATCH 62/64] renamed PrePostStageTriggerService -> preStageTriggerService --- .../{PrePostStageTriggerService.go => PreStageTriggerService.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pkg/deployment/trigger/devtronApps/{PrePostStageTriggerService.go => PreStageTriggerService.go} (100%) diff --git a/pkg/deployment/trigger/devtronApps/PrePostStageTriggerService.go b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go similarity index 100% rename from pkg/deployment/trigger/devtronApps/PrePostStageTriggerService.go rename to pkg/deployment/trigger/devtronApps/PreStageTriggerService.go From b34bfb1f59dbf4a4ddc9320133fe91903a845a11 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Fri, 9 Feb 2024 18:56:40 +0530 Subject: [PATCH 63/64] fix for rollback --- pkg/deployment/manifest/ManifestCreationService.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/deployment/manifest/ManifestCreationService.go b/pkg/deployment/manifest/ManifestCreationService.go index f53e587fb4..1134d8dc7f 100644 --- a/pkg/deployment/manifest/ManifestCreationService.go +++ b/pkg/deployment/manifest/ManifestCreationService.go @@ -340,7 +340,7 @@ func (impl *ManifestCreationServiceImpl) getEnvOverrideByTriggerType(overrideReq } //getting chart_ref by id _, span = otel.Tracer("orchestrator").Start(ctx, "chartRefRepository.FindByVersionAndName") - chartRefDto, err := impl.chartRefService.FindByVersionAndName(templateName, templateVersion) + chartRefDto, err := impl.chartRefService.FindByVersionAndName(templateVersion, templateName) span.End() if err != nil { impl.logger.Errorw("error in getting chartRef by version and name", "err", err, "version", templateVersion, "name", templateName) From 8c66440aabcaa4e40f222ec62c02677db439ca61 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Mon, 19 Feb 2024 15:50:26 +0530 Subject: [PATCH 64/64] updated common-lib version --- go.mod | 2 +- go.sum | 4 ++-- .../devtron-labs/common-lib/pubsub-lib/JetStreamUtil.go | 4 ++++ vendor/modules.txt | 2 +- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 11341ce0e8..7ba58f5a4b 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set v1.8.0 github.com/devtron-labs/authenticator v0.4.33 - github.com/devtron-labs/common-lib v0.0.14 + github.com/devtron-labs/common-lib v0.0.15 github.com/devtron-labs/protos v0.0.3-0.20240130061723-7b2e12ab0abb github.com/evanphx/json-patch v5.6.0+incompatible github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 diff --git a/go.sum b/go.sum index 2125d62346..8aa0f7b7ad 100644 --- a/go.sum +++ b/go.sum @@ -221,8 +221,8 @@ github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4 h1:YcpmyvADG github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM= github.com/devtron-labs/authenticator v0.4.33 h1:FpAV3ZgFluaRFcMwPpwxr/mwSipJ16XRvgABq3BzP5Y= github.com/devtron-labs/authenticator v0.4.33/go.mod h1:ozNfT8WcruiSgnUbyp48WVfc41++W6xYXhKFp67lNTU= -github.com/devtron-labs/common-lib v0.0.14 h1:7P0Z87Des8Bni+aVHhPITjzKITuCVOjwAHOgElrh/rk= -github.com/devtron-labs/common-lib v0.0.14/go.mod h1:95/DizzVXu1kHap/VwEvdxwgd+BvPVYc0bJzt8yqGDU= +github.com/devtron-labs/common-lib v0.0.15 h1:/t+x0G4oT1WWZfBtwYOp34EdbIx5rELZkI4N6vJ0uQc= +github.com/devtron-labs/common-lib v0.0.15/go.mod h1:95/DizzVXu1kHap/VwEvdxwgd+BvPVYc0bJzt8yqGDU= github.com/devtron-labs/protos v0.0.3-0.20240130061723-7b2e12ab0abb h1:CkfQQgZc950/hTPqtQSiHV2RmZgkBLGCzwR02FZYjAU= github.com/devtron-labs/protos v0.0.3-0.20240130061723-7b2e12ab0abb/go.mod h1:pjLjgoa1GzbkOkvbMyP4SAKsaiK7eG6GoQCNauG03JA= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= diff --git a/vendor/github.com/devtron-labs/common-lib/pubsub-lib/JetStreamUtil.go b/vendor/github.com/devtron-labs/common-lib/pubsub-lib/JetStreamUtil.go index c02ec09f69..9dcfc422ed 100644 --- a/vendor/github.com/devtron-labs/common-lib/pubsub-lib/JetStreamUtil.go +++ b/vendor/github.com/devtron-labs/common-lib/pubsub-lib/JetStreamUtil.go @@ -92,6 +92,9 @@ const ( PANIC_ON_PROCESSING_TOPIC string = "PANIC-ON-PROCESSING-TOPIC" PANIC_ON_PROCESSING_GROUP string = "PANIC-ON-PROCESSING-GROUP" PANIC_ON_PROCESSING_DURABLE string = "PANIC-ON-PROCESSING-DURABLE" + CD_STAGE_SUCCESS_EVENT_TOPIC string = "CD-STAGE-SUCCESS-EVENT" + CD_STAGE_SUCCESS_EVENT_GROUP string = "CD-STAGE-SUCCESS-EVENT-GROUP" + CD_STAGE_SUCCESS_EVENT_DURABLE string = "CD-STAGE-SUCCESS-EVENT-DURABLE" ) type NatsTopic struct { @@ -134,6 +137,7 @@ var natsTopicMapping = map[string]NatsTopic{ HELM_CHART_INSTALL_STATUS_TOPIC: {topicName: HELM_CHART_INSTALL_STATUS_TOPIC, streamName: ORCHESTRATOR_STREAM, queueName: HELM_CHART_INSTALL_STATUS_GROUP, consumerName: HELM_CHART_INSTALL_STATUS_DURABLE}, DEVTRON_CHART_INSTALL_TOPIC: {topicName: DEVTRON_CHART_INSTALL_TOPIC, streamName: ORCHESTRATOR_STREAM, queueName: DEVTRON_CHART_INSTALL_GROUP, consumerName: DEVTRON_CHART_INSTALL_DURABLE}, PANIC_ON_PROCESSING_TOPIC: {topicName: PANIC_ON_PROCESSING_TOPIC, streamName: ORCHESTRATOR_STREAM, queueName: PANIC_ON_PROCESSING_GROUP, consumerName: PANIC_ON_PROCESSING_DURABLE}, + CD_STAGE_SUCCESS_EVENT_TOPIC: {topicName: CD_STAGE_SUCCESS_EVENT_TOPIC, streamName: ORCHESTRATOR_STREAM, queueName: CD_STAGE_SUCCESS_EVENT_GROUP, consumerName: CD_STAGE_SUCCESS_EVENT_DURABLE}, } var NatsStreamWiseConfigMapping = map[string]NatsStreamConfig{ diff --git a/vendor/modules.txt b/vendor/modules.txt index 6237f51720..1d380ee83d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -373,7 +373,7 @@ github.com/devtron-labs/authenticator/jwt github.com/devtron-labs/authenticator/middleware github.com/devtron-labs/authenticator/oidc github.com/devtron-labs/authenticator/password -# github.com/devtron-labs/common-lib v0.0.14 +# github.com/devtron-labs/common-lib v0.0.15 ## explicit; go 1.20 github.com/devtron-labs/common-lib/blob-storage github.com/devtron-labs/common-lib/cloud-provider-identifier