Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
add build flag for scalaVersion
Browse files Browse the repository at this point in the history
@fqaiser94
fqaiser94 committed Feb 8, 2022
1 parent 9370164 commit ed6dc1a
Showing 9 changed files with 238 additions and 277 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/flink-ci.yml
View file
Original file line number Diff line number Diff line change
@@ -73,7 +73,7 @@ jobs:
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
restore-keys: ${{ runner.os }}-gradle
- run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
- run: ./gradlew -DsparkVersions= -DhiveVersions= -DflinkVersions=${{ matrix.flink }} :iceberg-flink:iceberg-flink-${{ matrix.flink }}:check :iceberg-flink:iceberg-flink-runtime-${{ matrix.flink }}:check -Pquick=true -x javadoc
- run: ./gradlew -DsparkVersions= -DscalaVersion= -DhiveVersions= -DflinkVersions=${{ matrix.flink }} :iceberg-flink:iceberg-flink-${{ matrix.flink }}:check :iceberg-flink:iceberg-flink-runtime-${{ matrix.flink }}:check -Pquick=true -x javadoc
- uses: actions/upload-artifact@v2
if: failure()
with:
4 changes: 2 additions & 2 deletions .github/workflows/hive-ci.yml
View file
Original file line number Diff line number Diff line change
@@ -68,7 +68,7 @@ jobs:
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
restore-keys: ${{ runner.os }}-gradle
- run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
- run: ./gradlew -DsparkVersions= -DhiveVersions=2 -DflinkVersions= -Pquick=true :iceberg-mr:check :iceberg-hive-runtime:check -x javadoc
- run: ./gradlew -DsparkVersions= -DscalaVersion= -DhiveVersions=2 -DflinkVersions= -Pquick=true :iceberg-mr:check :iceberg-hive-runtime:check -x javadoc
- uses: actions/upload-artifact@v2
if: failure()
with:
@@ -91,7 +91,7 @@ jobs:
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
restore-keys: ${{ runner.os }}-gradle
- run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
- run: ./gradlew -DsparkVersions= -DhiveVersions=3 -DflinkVersions= -Pquick=true :iceberg-hive3-orc-bundle:check :iceberg-hive3:check :iceberg-hive-runtime:check -x javadoc
- run: ./gradlew -DsparkVersions= -DscalaVersion= -DhiveVersions=3 -DflinkVersions= -Pquick=true :iceberg-hive3-orc-bundle:check :iceberg-hive3:check :iceberg-hive-runtime:check -x javadoc
- uses: actions/upload-artifact@v2
if: failure()
with:
4 changes: 2 additions & 2 deletions .github/workflows/java-ci.yml
View file
Original file line number Diff line number Diff line change
@@ -62,7 +62,7 @@ jobs:
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
restore-keys: ${{ runner.os }}-gradle
- run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
- run: ./gradlew check -DsparkVersions= -DhiveVersions= -DflinkVersions= -Pquick=true -x javadoc
- run: ./gradlew check -DsparkVersions= -DscalaVersion= -DhiveVersions= -DflinkVersions= -Pquick=true -x javadoc
- uses: actions/upload-artifact@v2
if: failure()
with:
@@ -77,7 +77,7 @@ jobs:
- uses: actions/setup-java@v1
with:
java-version: 8
- run: ./gradlew -DflinkVersions=1.12,1.13,1.14 -DsparkVersions=2.4,3.0,3.1,3.2 -DhiveVersions=2,3 build -x test -x javadoc -x integrationTest
- run: ./gradlew -DflinkVersions=1.12,1.13,1.14 -DsparkVersions=2.4,3.0,3.1,3.2 -DscalaVersion=2.12 -DhiveVersions=2,3 build -x test -x javadoc -x integrationTest

build-javadoc:
runs-on: ubuntu-latest
3 changes: 2 additions & 1 deletion .github/workflows/publish-snapshot.yml
View file
Original file line number Diff line number Diff line change
@@ -39,4 +39,5 @@ jobs:
java-version: 8
- run: |
./gradlew printVersion
./gradlew -DflinkVersions=1.12,1.13,1.14 -DsparkVersions=2.4,3.0,3.1,3.2 -DhiveVersions=2,3 publishApachePublicationToMavenRepository -PmavenUser=${{ secrets.NEXUS_USER }} -PmavenPassword=${{ secrets.NEXUS_PW }}
./gradlew -DflinkVersions=1.12,1.13,1.14 -DsparkVersions=2.4,3.0,3.1,3.2 -DscalaVersion=2.12 -DhiveVersions=2,3 publishApachePublicationToMavenRepository -PmavenUser=${{ secrets.NEXUS_USER }} -PmavenPassword=${{ secrets.NEXUS_PW }}
./gradlew -DflinkVersions= -DsparkVersions=3.2 -DscalaVersion=2.13 -DhiveVersions= publishApachePublicationToMavenRepository -PmavenUser=${{ secrets.NEXUS_USER }} -PmavenPassword=${{ secrets.NEXUS_PW }}
8 changes: 4 additions & 4 deletions .github/workflows/spark-ci.yml
View file
Original file line number Diff line number Diff line change
@@ -123,7 +123,7 @@ jobs:
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
restore-keys: ${{ runner.os }}-gradle
- run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
- run: ./gradlew -DsparkVersions=${{ matrix.spark }} -DhiveVersions= -DflinkVersions= :iceberg-spark:iceberg-spark-${{ matrix.spark }}_2.12:check :iceberg-spark:iceberg-spark-extensions-${{ matrix.spark }}_2.12:check :iceberg-spark:iceberg-spark-runtime-${{ matrix.spark }}_2.12:check -Pquick=true -x javadoc
- run: ./gradlew -DsparkVersions=${{ matrix.spark }} -DscalaVersion=2.12 -DhiveVersions= -DflinkVersions= :iceberg-spark:iceberg-spark-${{ matrix.spark }}_2.12:check :iceberg-spark:iceberg-spark-extensions-${{ matrix.spark }}_2.12:check :iceberg-spark:iceberg-spark-runtime-${{ matrix.spark }}_2.12:check -Pquick=true -x javadoc
- uses: actions/upload-artifact@v2
if: failure()
with:
@@ -135,8 +135,8 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
jvm: [ 8, 11 ]
spark: [ '3.2' ]
jvm: [8, 11]
spark: ['3.2']
env:
SPARK_LOCAL_IP: localhost
steps:
@@ -150,7 +150,7 @@ jobs:
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
restore-keys: ${{ runner.os }}-gradle
- run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
- run: ./gradlew -DsparkVersions=${{ matrix.spark }} -DhiveVersions= -DflinkVersions= :iceberg-spark:iceberg-spark-${{ matrix.spark }}_2.13:check :iceberg-spark:iceberg-spark-extensions-${{ matrix.spark }}_2.13:check :iceberg-spark:iceberg-spark-runtime-${{ matrix.spark }}_2.13:check -Pquick=true -x javadoc
- run: ./gradlew -DsparkVersions=${{ matrix.spark }} -DscalaVersion=2.13 -DhiveVersions= -DflinkVersions= :iceberg-spark:iceberg-spark-${{ matrix.spark }}_2.13:check :iceberg-spark:iceberg-spark-extensions-${{ matrix.spark }}_2.13:check :iceberg-spark:iceberg-spark-runtime-${{ matrix.spark }}_2.13:check -Pquick=true -x javadoc
- uses: actions/upload-artifact@v2
if: failure()
with:
2 changes: 2 additions & 0 deletions gradle.properties
View file
Original file line number Diff line number Diff line change
@@ -21,5 +21,7 @@ systemProp.defaultHiveVersions=2
systemProp.knownHiveVersions=2,3
systemProp.defaultSparkVersions=3.2
systemProp.knownSparkVersions=2.4,3.0,3.1,3.2
systemProp.defaultScalaVersion=2.12
systemProp.knownScalaVersions=2.12,2.13
org.gradle.parallel=true
org.gradle.jvmargs=-Xmx768m
4 changes: 2 additions & 2 deletions jmh.gradle
View file
Original file line number Diff line number Diff line change
@@ -22,6 +22,7 @@ if (jdkVersion != '8' && jdkVersion != '11') {
}

def sparkVersions = (System.getProperty("sparkVersions") != null ? System.getProperty("sparkVersions") : System.getProperty("defaultSparkVersions")).split(",")
def scalaVersion = System.getProperty("scalaVersion") != null ? System.getProperty("scalaVersion") : System.getProperty("defaultScalaVersion")
def jmhProjects = []

if (jdkVersion == '8' && sparkVersions.contains("2.4")) {
@@ -37,8 +38,7 @@ if (sparkVersions.contains("3.1")) {
}

if (sparkVersions.contains("3.2")) {
jmhProjects.add(project(":iceberg-spark:iceberg-spark-3.2_2.12"))
jmhProjects.add(project(":iceberg-spark:iceberg-spark-3.2_2.13"))
jmhProjects.add(project(":iceberg-spark:iceberg-spark-3.2_${scalaVersion}"))
}

configure(jmhProjects) {
35 changes: 16 additions & 19 deletions settings.gradle
View file
Original file line number Diff line number Diff line change
@@ -74,6 +74,13 @@ if (!knownSparkVersions.containsAll(sparkVersions)) {
throw new GradleException("Found unsupported Spark versions: " + (sparkVersions - knownSparkVersions))
}

List<String> knownScalaVersions = System.getProperty("knownScalaVersions").split(",")
String scalaVersion = System.getProperty("scalaVersion") != null ? System.getProperty("scalaVersion") : System.getProperty("defaultScalaVersion")

if (!knownScalaVersions.contains(scalaVersion)) {
throw new GradleException("Found unsupported Scala version: " + scalaVersion)
}

if (!flinkVersions.isEmpty()) {
include 'flink'
project(':flink').name = 'iceberg-flink'
@@ -131,25 +138,15 @@ if (sparkVersions.contains("3.1")) {
}

if (sparkVersions.contains("3.2")) {
include ':iceberg-spark:spark-3.2_2.12'
include ':iceberg-spark:spark-extensions-3.2_2.12'
include ':iceberg-spark:spark-runtime-3.2_2.12'
project(':iceberg-spark:spark-3.2_2.12').projectDir = file('spark/v3.2/spark')
project(':iceberg-spark:spark-3.2_2.12').name = 'iceberg-spark-3.2_2.12'
project(':iceberg-spark:spark-extensions-3.2_2.12').projectDir = file('spark/v3.2/spark-extensions')
project(':iceberg-spark:spark-extensions-3.2_2.12').name = 'iceberg-spark-extensions-3.2_2.12'
project(':iceberg-spark:spark-runtime-3.2_2.12').projectDir = file('spark/v3.2/spark-runtime')
project(':iceberg-spark:spark-runtime-3.2_2.12').name = 'iceberg-spark-runtime-3.2_2.12'

include ':iceberg-spark:spark-3.2_2.13'
include ':iceberg-spark:spark-extensions-3.2_2.13'
include ':iceberg-spark:spark-runtime-3.2_2.13'
project(':iceberg-spark:spark-3.2_2.13').projectDir = file('spark/v3.2/spark')
project(':iceberg-spark:spark-3.2_2.13').name = 'iceberg-spark-3.2_2.13'
project(':iceberg-spark:spark-extensions-3.2_2.13').projectDir = file('spark/v3.2/spark-extensions')
project(':iceberg-spark:spark-extensions-3.2_2.13').name = 'iceberg-spark-extensions-3.2_2.13'
project(':iceberg-spark:spark-runtime-3.2_2.13').projectDir = file('spark/v3.2/spark-runtime')
project(':iceberg-spark:spark-runtime-3.2_2.13').name = 'iceberg-spark-runtime-3.2_2.13'
include ":iceberg-spark:spark-3.2_${scalaVersion}"
include ":iceberg-spark:spark-extensions-3.2_${scalaVersion}"
include ":iceberg-spark:spark-runtime-3.2_${scalaVersion}"
project(":iceberg-spark:spark-3.2_${scalaVersion}").projectDir = file('spark/v3.2/spark')
project(":iceberg-spark:spark-3.2_${scalaVersion}").name = "iceberg-spark-3.2_${scalaVersion}"
project(":iceberg-spark:spark-extensions-3.2_${scalaVersion}").projectDir = file('spark/v3.2/spark-extensions')
project(":iceberg-spark:spark-extensions-3.2_${scalaVersion}").name = "iceberg-spark-extensions-3.2_${scalaVersion}"
project(":iceberg-spark:spark-runtime-3.2_${scalaVersion}").projectDir = file('spark/v3.2/spark-runtime')
project(":iceberg-spark:spark-runtime-3.2_${scalaVersion}").name = "iceberg-spark-runtime-3.2_${scalaVersion}"
}

// hive 3 depends on hive 2, so always add hive 2 if hive3 is enabled
Loading
Oops, something went wrong.

0 comments on commit ed6dc1a

Please sign in to comment.