Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/actions/setup-gradle/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ runs:
distribution: temurin
java-version: ${{ inputs.java-version }}
- name: Setup Gradle
uses: gradle/actions/setup-gradle@94baf225fe0a508e581a564467443d0e2379123b # v4.3.0
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you please use the git commit hash instead, to follow the Apache policy (see below)?

You MUST pin all external actions to the specific git hash (SHA1) of the action that has been reviewed for use by the project. For instance, you MUST pin foobar/baz-action@8843d7f92416211de9ebb963ff4ce28125932878.

reference: https://infra.apache.org/github-actions-policy.html

uses: gradle/actions/setup-gradle@ed408507eac070d1f99cc633dbcf757c94c7933a # v4.4.3
env:
GRADLE_BUILD_ACTION_CACHE_DEBUG_ENABLED: true
with:
Expand Down
198 changes: 126 additions & 72 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -29,22 +29,21 @@ buildscript {
}

plugins {
id 'com.github.ben-manes.versions' version '0.48.0'
id 'com.github.ben-manes.versions' version '0.52.0'
id 'idea'
id 'jacoco'
id 'java-library'
id 'org.owasp.dependencycheck' version '8.2.1'
id 'org.owasp.dependencycheck' version '12.1.3'
id 'org.nosphere.apache.rat' version "0.8.1"
id "io.swagger.core.v3.swagger-gradle-plugin" version "${swaggerVersion}"

id "com.github.spotbugs" version '6.2.3' apply false
id "com.github.spotbugs" version '6.2.5' apply false
id 'org.scoverage' version '8.0.3' apply false
id 'com.gradleup.shadow' version '8.3.6' apply false
id 'com.diffplug.spotless' version "6.25.0"
id 'com.gradleup.shadow' version '8.3.9' apply false
id 'com.diffplug.spotless' version "7.2.1"
}

ext {
gradleVersion = versions.gradle
minClientJavaVersion = 11
minNonClientJavaVersion = 17
modulesNeedingJava11 = [":clients", ":generator", ":streams", ":streams:test-utils", ":streams:examples", ":streams-scala", ":test-common:test-common-util"]
Expand Down Expand Up @@ -299,7 +298,7 @@ if (repo != null) {
} else {
rat.enabled = false
}
println("Starting build with version $version (commit id ${commitId == null ? "null" : commitId.take(8)}) using Gradle $gradleVersion, Java ${JavaVersion.current()} and Scala ${versions.scala}")
println("Starting build with version $version (commit id ${commitId == null ? "null" : commitId.take(8)}) using Gradle $versions.gradle, Java ${JavaVersion.current()} and Scala ${versions.scala}")
println("Build properties: ignoreFailures=$userIgnoreFailures, maxParallelForks=$maxTestForks, maxScalacThreads=$maxScalacThreads, maxTestRetries=$userMaxTestRetries")

subprojects {
Expand Down Expand Up @@ -330,6 +329,23 @@ subprojects {
tasks.register('uploadArchives').configure { dependsOn(publish) }
}

def submodulesToIncludeInDistribution = ['core', 'tools', 'trogdor', 'shell', 'api', 'runtime', 'transforms',
'json', 'file', 'basic-auth-extension', 'mirror', 'mirror-client',
'streams', 'streams-scala', 'test-utils', 'examples', 'tools-api']

def shouldModuleResolveRuntimeClasspath = (submodulesToIncludeInDistribution.contains(project.name))

if (shouldModuleResolveRuntimeClasspath) {
configurations.create('resolvedRuntimeClasspath') {
extendsFrom(configurations.runtimeClasspath)
canBeResolved = true
}
}

tasks.withType(AbstractTestTask).configureEach {
failOnNoDiscoveredTests = false
}

// apply the eclipse plugin only to subprojects that hold code. 'connect' is just a folder.
if (!project.name.equals('connect')) {
apply plugin: 'eclipse'
Expand Down Expand Up @@ -1170,71 +1186,6 @@ project(':core') {
standardOutput = new File(generatedDocsDir, "producer_metrics.html").newOutputStream()
}

task siteDocsTar(dependsOn: ['genProtocolErrorDocs', 'genProtocolTypesDocs', 'genProtocolApiKeyDocs', 'genProtocolMessageDocs',
'genAdminClientConfigDocs', 'genProducerConfigDocs', 'genConsumerConfigDocs',
'genKafkaConfigDocs', 'genTopicConfigDocs', 'genGroupConfigDocs',
':connect:runtime:genConnectConfigDocs', ':connect:runtime:genConnectTransformationDocs',
':connect:runtime:genConnectPredicateDocs',
':connect:runtime:genSinkConnectorConfigDocs', ':connect:runtime:genSourceConnectorConfigDocs',
':streams:genStreamsConfigDocs', 'genConsumerMetricsDocs', 'genProducerMetricsDocs',
':connect:runtime:genConnectMetricsDocs', ':connect:runtime:genConnectOpenAPIDocs',
':connect:mirror:genMirrorSourceConfigDocs', ':connect:mirror:genMirrorCheckpointConfigDocs',
':connect:mirror:genMirrorHeartbeatConfigDocs', ':connect:mirror:genMirrorConnectorConfigDocs',
':storage:genRemoteLogManagerConfigDoc', ':storage:genRemoteLogMetadataManagerConfigDoc'], type: Tar) {
archiveClassifier = 'site-docs'
compression = Compression.GZIP
from project.file("$rootDir/docs")
into 'site-docs'
duplicatesStrategy 'exclude'
}

tasks.create(name: "releaseTarGz", dependsOn: configurations.archives.artifacts, type: Tar) {
into "kafka_${versions.baseScala}-${archiveVersion.get()}"
compression = Compression.GZIP
from(project.file("$rootDir/bin")) { into "bin/" }
from(project.file("$rootDir/config")) { into "config/" }
from(project.file("$rootDir/licenses")) { into "licenses/" }
from "$rootDir/LICENSE-binary" rename {String filename -> filename.replace("-binary", "")}
from "$rootDir/NOTICE-binary" rename {String filename -> filename.replace("-binary", "")}
from(configurations.runtimeClasspath) { into("libs/") }
from(configurations.archives.artifacts.files) { into("libs/") }
from(configurations.releaseOnly) { into("libs/") }
from(project.siteDocsTar) { into("site-docs/") }
from(project(':tools').jar) { into("libs/") }
from(project(':tools').configurations.runtimeClasspath) { into("libs/") }
from(project(':trogdor').jar) { into("libs/") }
from(project(':trogdor').configurations.runtimeClasspath) { into("libs/") }
from(project(':shell').jar) { into("libs/") }
from(project(':shell').configurations.runtimeClasspath) { into("libs/") }
from(project(':connect:api').jar) { into("libs/") }
from(project(':connect:api').configurations.runtimeClasspath) { into("libs/") }
from(project(':connect:runtime').jar) { into("libs/") }
from(project(':connect:runtime').configurations.runtimeClasspath) { into("libs/") }
from(project(':connect:transforms').jar) { into("libs/") }
from(project(':connect:transforms').configurations.runtimeClasspath) { into("libs/") }
from(project(':connect:json').jar) { into("libs/") }
from(project(':connect:json').configurations.runtimeClasspath) { into("libs/") }
from(project(':connect:file').jar) { into("libs/") }
from(project(':connect:file').configurations.runtimeClasspath) { into("libs/") }
from(project(':connect:basic-auth-extension').jar) { into("libs/") }
from(project(':connect:basic-auth-extension').configurations.runtimeClasspath) { into("libs/") }
from(project(':connect:mirror').jar) { into("libs/") }
from(project(':connect:mirror').configurations.runtimeClasspath) { into("libs/") }
from(project(':connect:mirror-client').jar) { into("libs/") }
from(project(':connect:mirror-client').configurations.runtimeClasspath) { into("libs/") }
from(project(':streams').jar) { into("libs/") }
from(project(':streams').configurations.runtimeClasspath) { into("libs/") }
from(project(':streams:streams-scala').jar) { into("libs/") }
from(project(':streams:streams-scala').configurations.runtimeClasspath) { into("libs/") }
from(project(':streams:test-utils').jar) { into("libs/") }
from(project(':streams:test-utils').configurations.runtimeClasspath) { into("libs/") }
from(project(':streams:examples').jar) { into("libs/") }
from(project(':streams:examples').configurations.runtimeClasspath) { into("libs/") }
from(project(':tools:tools-api').jar) { into("libs/") }
from(project(':tools:tools-api').configurations.runtimeClasspath) { into("libs/") }
duplicatesStrategy 'exclude'
}

jar {
dependsOn copyDependantLibs
}
Expand Down Expand Up @@ -3905,6 +3856,109 @@ project(':connect:test-plugins') {
}
}

project(':distribution') {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just wondering - does this change matter for the upgrade from gradle 8 to 9

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@dejan2609 Curious about your thoughts here - would adding a new module be necessary, or could we simply continue handling the distributions within the core module?

base {
archivesName = "kafka_${versions.baseScala}"
}

configurations {
includeIntoDistribution
}

dependencies {
includeIntoDistribution project(path: ':core', configuration: 'releaseOnly')
includeIntoDistribution project(path: ':core', configuration: 'resolvedRuntimeClasspath')
includeIntoDistribution project(path: ':tools', configuration: 'resolvedRuntimeClasspath')
includeIntoDistribution project(path: ':trogdor', configuration: 'resolvedRuntimeClasspath')
includeIntoDistribution project(path: ':shell', configuration: 'resolvedRuntimeClasspath')
includeIntoDistribution project(path: ':connect:api', configuration: 'resolvedRuntimeClasspath')
includeIntoDistribution project(path: ':connect:runtime', configuration: 'resolvedRuntimeClasspath')
includeIntoDistribution project(path: ':connect:transforms', configuration: 'resolvedRuntimeClasspath')
includeIntoDistribution project(path: ':connect:json', configuration: 'resolvedRuntimeClasspath')
includeIntoDistribution project(path: ':connect:file', configuration: 'resolvedRuntimeClasspath')
includeIntoDistribution project(path: ':connect:basic-auth-extension', configuration: 'resolvedRuntimeClasspath')
includeIntoDistribution project(path: ':connect:mirror', configuration: 'resolvedRuntimeClasspath')
includeIntoDistribution project(path: ':connect:mirror-client', configuration: 'resolvedRuntimeClasspath')
includeIntoDistribution project(path: ':streams', configuration: 'resolvedRuntimeClasspath')
includeIntoDistribution project(path: ':streams:streams-scala', configuration: 'resolvedRuntimeClasspath')
includeIntoDistribution project(path: ':streams:test-utils', configuration: 'resolvedRuntimeClasspath')
includeIntoDistribution project(path: ':streams:examples', configuration: 'resolvedRuntimeClasspath')
includeIntoDistribution project(path: ':tools:tools-api', configuration: 'resolvedRuntimeClasspath')
}

task siteDocsTar(dependsOn: [':core:genProtocolErrorDocs', ':core:genProtocolTypesDocs', ':core:genProtocolApiKeyDocs', ':core:genProtocolMessageDocs',
':core:genAdminClientConfigDocs', ':core:genProducerConfigDocs', ':core:genConsumerConfigDocs',
':core:genKafkaConfigDocs', ':core:genTopicConfigDocs', ':core:genGroupConfigDocs',
':connect:runtime:genConnectConfigDocs', ':connect:runtime:genConnectTransformationDocs',
':connect:runtime:genConnectPredicateDocs',
':connect:runtime:genSinkConnectorConfigDocs', ':connect:runtime:genSourceConnectorConfigDocs',
':streams:genStreamsConfigDocs', ':core:genConsumerMetricsDocs', ':core:genProducerMetricsDocs',
':connect:runtime:genConnectMetricsDocs', ':connect:runtime:genConnectOpenAPIDocs',
':connect:mirror:genMirrorSourceConfigDocs', ':connect:mirror:genMirrorCheckpointConfigDocs',
':connect:mirror:genMirrorHeartbeatConfigDocs', ':connect:mirror:genMirrorConnectorConfigDocs',
':storage:genRemoteLogManagerConfigDoc', ':storage:genRemoteLogMetadataManagerConfigDoc'], type: Tar) {
archiveClassifier = 'site-docs'
compression = Compression.GZIP
from project.file("$rootDir/docs")
into 'site-docs'
destinationDirectory = project(':core').layout.buildDirectory.dir('distributions')
duplicatesStrategy 'exclude'
preserveFileTimestamps = true
}

tasks.create(name: "releaseTarGz", dependsOn: project(':core').configurations.archives.artifacts, type: Tar) {
into "kafka_${versions.baseScala}-${archiveVersion.get()}"
compression = Compression.GZIP
from(project.file("$rootDir/bin")) { into "bin/" }
from(project.file("$rootDir/config")) { into "config/" }
from(project.file("$rootDir/licenses")) { into "licenses/" }
from "$rootDir/LICENSE-binary" rename {String filename -> filename.replace("-binary", "")}
from "$rootDir/NOTICE-binary" rename {String filename -> filename.replace("-binary", "")}
from(project.siteDocsTar) { into("site-docs/") }
from(configurations.includeIntoDistribution) { into("libs/") }
from(project(':core').configurations.archives.artifacts.files) { into("libs/") }
from(project(':tools').jar) { into("libs/") }
from(project(':trogdor').jar) { into("libs/") }
from(project(':shell').jar) { into("libs/") }
from(project(':connect:api').jar) { into("libs/") }
from(project(':connect:runtime').jar) { into("libs/") }
from(project(':connect:transforms').jar) { into("libs/") }
from(project(':connect:json').jar) { into("libs/") }
from(project(':connect:file').jar) { into("libs/") }
from(project(':connect:basic-auth-extension').jar) { into("libs/") }
from(project(':connect:mirror').jar) { into("libs/") }
from(project(':connect:mirror-client').jar) { into("libs/") }
from(project(':streams').jar) { into("libs/") }
from(project(':streams:streams-scala').jar) { into("libs/") }
from(project(':streams:test-utils').jar) { into("libs/") }
from(project(':streams:examples').jar) { into("libs/") }
from(project(':tools:tools-api').jar) { into("libs/") }
destinationDirectory = project(':core').layout.buildDirectory.dir('distributions')
duplicatesStrategy 'exclude'
preserveFileTimestamps = true
filePermissions {
user {
read = true
write = true
}
group.read = true
other.read = true
}
dirPermissions {
unix('rwxr-xr-x')
}
eachFile {
if (name.endsWith( ".sh")) {
permissions {
user.execute = true
group.execute = true
other.execute = true
}
}
}
}
}

task aggregatedJavadoc(type: Javadoc, dependsOn: compileJava) {
def projectsWithJavadoc = subprojects.findAll { it.javadoc.enabled }
source = projectsWithJavadoc.collect { it.sourceSets.main.allJava }
Expand Down
6 changes: 3 additions & 3 deletions gradle/dependencies.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,8 @@ versions += [
commonsLang: "3.18.0",
commonsValidator: "1.9.0",
classgraph: "4.8.179",
gradle: "8.14.3",
grgit: "4.1.1",
gradle: "9.1.0",
grgit: "5.3.0",
httpclient: "4.5.14",
jackson: "2.19.0",
jacoco: "0.8.13",
Expand Down Expand Up @@ -126,7 +126,7 @@ versions += [
snappy: "1.1.10.7",
spotbugs: "4.9.4",
mockOAuth2Server: "2.2.1",
zinc: "1.9.2",
zinc: "1.10.8",
// When updating the zstd version, please do as well in docker/native/native-image-configs/resource-config.json
// Also make sure the compression levels in org.apache.kafka.common.record.CompressionType are still valid
zstd: "1.5.6-10",
Expand Down
4 changes: 2 additions & 2 deletions gradle/wrapper/gradle-wrapper.properties
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionSha256Sum=bd71102213493060956ec229d946beee57158dbd89d0e62b91bca0fa2c5f3531
distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip
distributionSha256Sum=a17ddd85a26b6a7f5ddb71ff8b05fc5104c0202c6e64782429790c933686c806
distributionUrl=https\://services.gradle.org/distributions/gradle-9.1.0-bin.zip
networkTimeout=10000
validateDistributionUrl=true
zipStoreBase=GRADLE_USER_HOME
Expand Down
41 changes: 20 additions & 21 deletions gradlew
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!/bin/sh

#
# Copyright © 2015-2021 the original authors.
# Copyright © 2015 the original authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
Expand All @@ -15,6 +15,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
# SPDX-License-Identifier: Apache-2.0
#

##############################################################################
#
Expand Down Expand Up @@ -55,7 +57,7 @@
# Darwin, MinGW, and NonStop.
#
# (3) This script is generated from the Groovy template
# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
# within the Gradle project.
#
# You can find Gradle at https://github.com/gradle/gradle/.
Expand Down Expand Up @@ -84,7 +86,7 @@ done
# shellcheck disable=SC2034
APP_BASE_NAME=${0##*/}
# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit
APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit

# Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD=maximum
Expand Down Expand Up @@ -113,20 +115,6 @@ case "$( uname )" in #(
esac


# Loop in case we encounter an error.
for attempt in 1 2 3; do
if [ ! -e "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" ]; then
if ! curl -s -S --retry 3 -L -o "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" "https://raw.githubusercontent.com/gradle/gradle/v8.14.3/gradle/wrapper/gradle-wrapper.jar"; then
rm -f "$APP_HOME/gradle/wrapper/gradle-wrapper.jar"
# Pause for a bit before looping in case the server throttled us.
sleep 5
continue
fi
fi
done

CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar


# Determine the Java command to use to start the JVM.
if [ -n "$JAVA_HOME" ] ; then
Expand Down Expand Up @@ -183,7 +171,6 @@ fi
# For Cygwin or MSYS, switch paths to Windows format before running java
if "$cygwin" || "$msys" ; then
APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" )

JAVACMD=$( cygpath --unix "$JAVACMD" )

Expand Down Expand Up @@ -212,19 +199,31 @@ if "$cygwin" || "$msys" ; then
fi



# Loop in case we encounter an error.
for attempt in 1 2 3; do
if [ ! -e "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" ]; then
if ! curl -s -S --retry 3 -L -o "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" "https://raw.githubusercontent.com/gradle/gradle/v9.1.0/gradle/wrapper/gradle-wrapper.jar"; then
rm -f "$APP_HOME/gradle/wrapper/gradle-wrapper.jar"
# Pause for a bit before looping in case the server throttled us.
sleep 5
continue
fi
fi
done

# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'

# Collect all arguments for the java command:
# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
# and any embedded shellness will be escaped.
# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
# treated as '${Hostname}' itself on the command line.

set -- \
"-Dorg.gradle.appname=$APP_BASE_NAME" \
-classpath "$CLASSPATH" \
org.gradle.wrapper.GradleWrapperMain \
-jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \
"$@"

# Stop when "xargs" is not available.
Expand Down
2 changes: 2 additions & 0 deletions settings.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ include 'clients',
'connect:transforms',
'coordinator-common',
'core',
'distribution',
'examples',
'generator',
'group-coordinator',
Expand Down Expand Up @@ -127,5 +128,6 @@ include 'clients',
'test-common:test-common-runtime'

project(":storage:api").name = "storage-api"
project(":distribution").projectDir.mkdirs()
rootProject.name = 'kafka'

Loading
Loading