feat: add auth support for downloading OCI artifacts (#3915)

Author: knqyf263

docs/docs/vulnerability/db.md

@@ -0,0 +1,38 @@
+# Database
+Trivy uses two types of databases for vulnerability detection:
+
+- Vulnerability Database
+- Java Index Database
+
+This page provides detailed information about these databases.
+
+## Vulnerability Database
+Trivy utilizes a database containing vulnerability information.
+This database is built every six hours on [GitHub](https://github.com/aquasecurity/trivy-db) and is distributed via [GitHub Container registry (GHCR)](https://ghcr.io/aquasecurity/trivy-db).
+The database is cached and updated as needed.
+As Trivy updates the database automatically during execution, users don't need to be concerned about it.
+
+For CLI flags related to the database, please refer to [this page](./examples/db.md).
+
+### Private Hosting
+If you host the database on your own OCI registry, you can specify a different repository with the `--db-repository` flag.
+The default is `ghcr.io/aquasecurity/trivy-db`.
+
+```shell
+$ trivy image --db-repository YOUR_REPO YOUR_IMAGE
+```
+
+If authentication is required, it can be configured in the same way as for private images.
+Please refer to [the documentation](../advanced/private-registries/index.md) for more details.
+
+## Java Index Database
+This database is only downloaded when scanning JAR files so that Trivy can identify the groupId, artifactId, and version of JAR files.
+It is built once a day on [GitHub](https://github.com/aquasecurity/trivy-java-db) and distributed via [GitHub Container registry (GHCR)](https://ghcr.io/aquasecurity/trivy-java-db).
+Like the vulnerability database, it is automatically downloaded and updated when needed, so users don't need to worry about it.
+
+### Private Hosting
+If you host the database on your own OCI registry, you can specify a different repository with the `--java-db-repository` flag.
+The default is `ghcr.io/aquasecurity/trivy-java-db`.
+
+If authentication is required, you need to run `docker login YOUR_REGISTRY`.
+Currently, specifying a username and password is not supported.

docs/docs/vulnerability/examples/db.md

@@ -1,9 +1,7 @@
 # Vulnerability DB
 
 ## Skip update of vulnerability DB
-`Trivy` downloads its vulnerability database every 12 hours when it starts operating.
-This is usually fast, as the size of the DB is only 10~30MB.
-But if you want to skip even that, use the `--skip-db-update` option.
+If you want to skip downloading the vulnerability database, use the `--skip-db-update` option.
 
 ```
 $ trivy image --skip-db-update python:3.4-alpine3.9

mkdocs.yml

@@ -52,6 +52,7 @@ nav:
               - OS Packages: docs/vulnerability/detection/os.md
               - Language-specific Packages: docs/vulnerability/detection/language.md
               - Data Sources: docs/vulnerability/detection/data-source.md
+          - Database: docs/vulnerability/db.md
           - Examples:
               - Vulnerability Filtering: docs/vulnerability/examples/filter.md
               - Report Formats: docs/vulnerability/examples/report.md

pkg/commands/app.go

@@ -759,7 +759,7 @@ func NewModuleCommand(globalFlags *flag.GlobalFlagGroup) *cobra.Command {
 				if err != nil {
 					return xerrors.Errorf("flag error: %w", err)
 				}
-				return module.Install(cmd.Context(), opts.ModuleDir, repo, opts.Quiet, opts.Insecure)
+				return module.Install(cmd.Context(), opts.ModuleDir, repo, opts.Quiet, opts.Remote())
 			},
 		},
 		&cobra.Command{

pkg/commands/artifact/run.go

@@ -19,7 +19,6 @@ import (
 	"github.com/aquasecurity/trivy/pkg/fanal/analyzer/config"
 	"github.com/aquasecurity/trivy/pkg/fanal/artifact"
 	"github.com/aquasecurity/trivy/pkg/fanal/cache"
-	ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
 	"github.com/aquasecurity/trivy/pkg/flag"
 	"github.com/aquasecurity/trivy/pkg/javadb"
 	"github.com/aquasecurity/trivy/pkg/log"
@@ -315,7 +314,7 @@ func (r *runner) initDB(opts flag.Options) error {
 
 	// download the database file
 	noProgress := opts.Quiet || opts.NoProgress
-	if err := operation.DownloadDB(opts.AppVersion, opts.CacheDir, opts.DBRepository, noProgress, opts.Insecure, opts.SkipDBUpdate); err != nil {
+	if err := operation.DownloadDB(opts.AppVersion, opts.CacheDir, opts.DBRepository, noProgress, opts.SkipDBUpdate, opts.Remote()); err != nil {
 		return err
 	}
 
@@ -610,13 +609,7 @@ func initScannerConfig(opts flag.Options, cacheClient cache.Cache) (ScannerConfi
 		}
 	}
 
-	remoteOption := ftypes.RemoteOptions{
-		Credentials:   opts.Credentials,
-		RegistryToken: opts.RegistryToken,
-		Insecure:      opts.Insecure,
-		Platform:      opts.Platform,
-		AWSRegion:     opts.AWSOptions.Region,
-	}
+	remoteOpts := opts.Remote()
 
 	return ScannerConfig{
 		Target:             target,
@@ -643,8 +636,8 @@ func initScannerConfig(opts flag.Options, cacheClient cache.Cache) (ScannerConfi
 			Slow:              opts.Slow,
 			AWSRegion:         opts.Region,
 
-			// For container registries
-			RemoteOptions: remoteOption,
+			// For OCI registries
+			RemoteOptions: remoteOpts,
 
 			// For misconfiguration scanning
 			MisconfScannerOption: configScannerOptions,

pkg/commands/operation/operation.go

@@ -15,6 +15,7 @@ import (
 	"github.com/aquasecurity/trivy-db/pkg/metadata"
 	"github.com/aquasecurity/trivy/pkg/db"
 	"github.com/aquasecurity/trivy/pkg/fanal/cache"
+	"github.com/aquasecurity/trivy/pkg/fanal/types"
 	"github.com/aquasecurity/trivy/pkg/flag"
 	"github.com/aquasecurity/trivy/pkg/log"
 	"github.com/aquasecurity/trivy/pkg/policy"
@@ -101,8 +102,8 @@ func (c Cache) ClearArtifacts() error {
 }
 
 // DownloadDB downloads the DB
-func DownloadDB(appVersion, cacheDir, dbRepository string, quiet, insecure, skipUpdate bool) error {
-	client := db.NewClient(cacheDir, quiet, insecure, db.WithDBRepository(dbRepository))
+func DownloadDB(appVersion, cacheDir, dbRepository string, quiet, skipUpdate bool, opt types.RemoteOptions) error {
+	client := db.NewClient(cacheDir, quiet, db.WithDBRepository(dbRepository))
 	ctx := context.Background()
 	needsUpdate, err := client.NeedsUpdate(appVersion, skipUpdate)
 	if err != nil {
@@ -113,7 +114,7 @@ func DownloadDB(appVersion, cacheDir, dbRepository string, quiet, insecure, skip
 		log.Logger.Info("Need to update DB")
 		log.Logger.Infof("DB Repository: %s", dbRepository)
 		log.Logger.Info("Downloading DB...")
-		if err = client.Download(ctx, cacheDir); err != nil {
+		if err = client.Download(ctx, cacheDir, opt); err != nil {
 			return xerrors.Errorf("failed to download vulnerability DB: %w", err)
 		}
 	}
@@ -145,7 +146,7 @@ func InitBuiltinPolicies(ctx context.Context, cacheDir string, quiet, skipUpdate
 
 	needsUpdate := false
 	if !skipUpdate {
-		needsUpdate, err = client.NeedsUpdate()
+		needsUpdate, err = client.NeedsUpdate(ctx)
 		if err != nil {
 			return nil, xerrors.Errorf("unable to check if built-in policies need to be updated: %w", err)
 		}

pkg/commands/server/run.go

@@ -35,7 +35,7 @@ func Run(ctx context.Context, opts flag.Options) (err error) {
 
 	// download the database file
 	if err = operation.DownloadDB(opts.AppVersion, opts.CacheDir, opts.DBRepository,
-		true, opts.Insecure, opts.SkipDBUpdate); err != nil {
+		true, opts.SkipDBUpdate, opts.Remote()); err != nil {
 		return err
 	}
 
@@ -57,6 +57,7 @@ func Run(ctx context.Context, opts flag.Options) (err error) {
 	}
 	m.Register()
 
-	server := rpcServer.NewServer(opts.AppVersion, opts.Listen, opts.CacheDir, opts.Token, opts.TokenHeader, opts.DBRepository)
-	return server.ListenAndServe(cache, opts.Insecure, opts.SkipDBUpdate)
+	server := rpcServer.NewServer(opts.AppVersion, opts.Listen, opts.CacheDir, opts.Token, opts.TokenHeader,
+		opts.DBRepository, opts.Remote())
+	return server.ListenAndServe(cache, opts.SkipDBUpdate)
 }

pkg/db/db.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/aquasecurity/trivy-db/pkg/db"
 	"github.com/aquasecurity/trivy-db/pkg/metadata"
+	"github.com/aquasecurity/trivy/pkg/fanal/types"
 	"github.com/aquasecurity/trivy/pkg/log"
 	"github.com/aquasecurity/trivy/pkg/oci"
 )
@@ -24,7 +25,7 @@ const (
 // Operation defines the DB operations
 type Operation interface {
 	NeedsUpdate(cliVersion string, skip bool) (need bool, err error)
-	Download(ctx context.Context, dst string) (err error)
+	Download(ctx context.Context, dst string, opt types.RemoteOptions) (err error)
 }
 
 type options struct {
@@ -61,14 +62,13 @@ func WithClock(clock clock.Clock) Option {
 type Client struct {
 	*options
 
-	cacheDir              string
-	metadata              metadata.Client
-	quiet                 bool
-	insecureSkipTLSVerify bool
+	cacheDir string
+	metadata metadata.Client
+	quiet    bool
 }
 
 // NewClient is the factory method for DB client
-func NewClient(cacheDir string, quiet, insecure bool, opts ...Option) *Client {
+func NewClient(cacheDir string, quiet bool, opts ...Option) *Client {
 	o := &options{
 		clock:        clock.RealClock{},
 		dbRepository: defaultDBRepository,
@@ -79,11 +79,10 @@ func NewClient(cacheDir string, quiet, insecure bool, opts ...Option) *Client {
 	}
 
 	return &Client{
-		options:               o,
-		cacheDir:              cacheDir,
-		metadata:              metadata.NewClient(cacheDir),
-		quiet:                 quiet,
-		insecureSkipTLSVerify: insecure, // insecure skip for download DB
+		options:  o,
+		cacheDir: cacheDir,
+		metadata: metadata.NewClient(cacheDir),
+		quiet:    quiet,
 	}
 }
 
@@ -144,18 +143,18 @@ func (c *Client) isNewDB(meta metadata.Metadata) bool {
 }
 
 // Download downloads the DB file
-func (c *Client) Download(ctx context.Context, dst string) error {
+func (c *Client) Download(ctx context.Context, dst string, opt types.RemoteOptions) error {
 	// Remove the metadata file under the cache directory before downloading DB
 	if err := c.metadata.Delete(); err != nil {
 		log.Logger.Debug("no metadata file")
 	}
 
-	art, err := c.initOCIArtifact()
+	art, err := c.initOCIArtifact(opt)
 	if err != nil {
 		return xerrors.Errorf("OCI artifact error: %w", err)
 	}
 
-	if err = art.Download(ctx, db.Dir(dst)); err != nil {
+	if err = art.Download(ctx, db.Dir(dst), oci.DownloadOption{MediaType: dbMediaType}); err != nil {
 		return xerrors.Errorf("database download error: %w", err)
 	}
 
@@ -184,13 +183,13 @@ func (c *Client) updateDownloadedAt(dst string) error {
 	return nil
 }
 
-func (c *Client) initOCIArtifact() (*oci.Artifact, error) {
+func (c *Client) initOCIArtifact(opt types.RemoteOptions) (*oci.Artifact, error) {
 	if c.artifact != nil {
 		return c.artifact, nil
 	}
 
 	repo := fmt.Sprintf("%s:%d", c.dbRepository, db.SchemaVersion)
-	art, err := oci.NewArtifact(repo, dbMediaType, "", c.quiet, c.insecureSkipTLSVerify)
+	art, err := oci.NewArtifact(repo, c.quiet, opt)
 	if err != nil {
 		var terr *transport.Error
 		if errors.As(err, &terr) {

pkg/db/db_test.go

@@ -18,6 +18,7 @@ import (
 	tdb "github.com/aquasecurity/trivy-db/pkg/db"
 	"github.com/aquasecurity/trivy-db/pkg/metadata"
 	"github.com/aquasecurity/trivy/pkg/db"
+	ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
 	"github.com/aquasecurity/trivy/pkg/oci"
 )
 
@@ -152,7 +153,7 @@ func TestClient_NeedsUpdate(t *testing.T) {
 				require.NoError(t, err)
 			}
 
-			client := db.NewClient(cacheDir, true, false, db.WithClock(tt.clock))
+			client := db.NewClient(cacheDir, true, db.WithClock(tt.clock))
 			needsUpdate, err := client.NeedsUpdate("test", tt.skip)
 
 			switch {
@@ -218,11 +219,14 @@ func TestClient_Download(t *testing.T) {
 			}, nil)
 
 			// Mock OCI artifact
-			art, err := oci.NewArtifact("db", mediaType, "", true, false, oci.WithImage(img))
+			opt := ftypes.RemoteOptions{
+				Insecure: false,
+			}
+			art, err := oci.NewArtifact("db", true, opt, oci.WithImage(img))
 			require.NoError(t, err)
 
-			client := db.NewClient(cacheDir, true, false, db.WithOCIArtifact(art), db.WithClock(timeDownloadedAt))
-			err = client.Download(context.Background(), cacheDir)
+			client := db.NewClient(cacheDir, true, db.WithOCIArtifact(art), db.WithClock(timeDownloadedAt))
+			err = client.Download(context.Background(), cacheDir, opt)
 			if tt.wantErr != "" {
 				require.Error(t, err)
 				assert.Contains(t, err.Error(), tt.wantErr)