build.gradle

buildscript {
    repositories {
        mavenCentral()
    }
    dependencies {
        classpath 'org.springframework.boot:spring-boot-gradle-plugin:1.5.7.RELEASE'
    }
}

plugins {
    id 'com.github.lkishalmi.gatling' version '0.4.1' apply false
    id 'com.github.johnrengelman.shadow' version '2.0.1' apply false
    id 'org.hidetake.ssh' version '2.9.0'
}

remotes {
    hadoopMaster {
        host = 'hadoop-1'
        user = 'vagrant'
//        identity = file("${System.getProperty('user.home')}/.vagrant.d/insecure_private_key")
        identity = file("/Users/markus/.vagrant.d/boxes/markush81-VAGRANTSLASH-centos7-vbox-guestadditions/1.0.3/virtualbox/vagrant_private_key")
    }
}

ssh {
    settings {
        knownHosts = allowAnyHosts
    }
}

allprojects {
    apply plugin: 'idea'

    repositories {
        mavenCentral()
        maven {
            url "https://repository.apache.org/content/repositories/snapshots/"
        }
    }
}

subprojects {
    apply plugin: 'java'

    sourceCompatibility = JavaVersion.VERSION_1_8
    targetCompatibility = JavaVersion.VERSION_1_8

    task sourcesJar(type: Jar, dependsOn: classes) {
        classifier = 'sources'
        from sourceSets.main.allSource
    }

    task javadocJar(type: Jar, dependsOn: javadoc) {
        classifier = 'javadoc'
        from javadoc.destinationDir
    }

    artifacts {
        archives sourcesJar
        archives javadocJar
    }

    dependencies {
        compile 'com.google.guava:guava:21.+'

        // logging
        compile 'org.slf4j:slf4j-api:1.7.+'
        compile 'ch.qos.logback:logback-classic:1.1.+'

        // testing
        testCompile 'junit:junit:4.+'
        testCompile 'org.hamcrest:hamcrest-library:1.+'
    }
}

project(':external') {
    apply plugin: 'org.springframework.boot'
    apply plugin: 'scala'
    apply plugin: 'com.github.lkishalmi.gatling'

    dependencies {
        //scala because of gatling
        compile 'org.scala-lang:scala-library:2.11.+'

        //application
        compile project(':legacy')
        compile 'org.springframework.boot:spring-boot-starter-web'
        compile 'org.springframework.boot:spring-boot-starter-hateoas'
        compile 'org.springframework.boot:spring-boot-starter-actuator'
        compile 'org.springframework.kafka:spring-kafka:1.2.1.RELEASE'
        compile 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.8.+'
        //devtools
        compile 'org.springframework.boot:spring-boot-devtools'
        //database
        compile 'org.hibernate:hibernate-java8'
        compile 'com.h2database:h2:1.4.193'

        //testing
        testCompile 'org.springframework.boot:spring-boot-starter-test'
        testCompile ('org.springframework.kafka:spring-kafka-test:1.3.0.RELEASE') {
            exclude module: 'log4j'
        }
        testCompile 'com.jayway.jsonpath:json-path-assert:2.2.0'
        testCompile 'org.exparity:hamcrest-date:2.0.4'
        testCompile 'org.awaitility:awaitility:2.0.0'
        testRuntime 'javax.xml.bind:jaxb-api:2.3.0'

        //scala test
        gatlingCompile 'org.scalatest:scalatest_2.11:3.0.4'
        gatlingRuntime 'com.h2database:h2:1.4.193'
    }
    build.dependsOn gatlingClasses

    bootRepackage {
        excludeDevtools = true
    }

    gatling {
        toolVersion = '2.2.3'
    }

    task h2console(type:JavaExec) {
        main = 'org.h2.tools.Console'
        classpath = sourceSets.main.runtimeClasspath
    }
}

project(':legacy') {
    dependencies {
        compile 'org.springframework.boot:spring-boot-starter-data-jpa:1.5.7.RELEASE'
    }
}

project(':flink') {
    apply plugin: 'scala'
    apply plugin: 'com.github.johnrengelman.shadow'

    sourceSets {
        flink
        main {
            compileClasspath = compileClasspath + configurations.flinkCompileClasspath
            runtimeClasspath = runtimeClasspath + configurations.flinkCompileClasspath
        }
    }

    dependencies {
        //scala
        compile 'org.scala-lang:scala-library:2.11.+'

        //flink
        compile 'org.apache.flink:flink-scala_2.11:1.4-SNAPSHOT'
        compile 'org.apache.flink:flink-streaming-scala_2.11:1.4-SNAPSHOT'
        compile 'org.apache.flink:flink-clients_2.11:1.4-SNAPSHOT'
        flinkCompile 'org.apache.flink:flink-connector-kafka-0.11_2.11:1.4-SNAPSHOT'
    }

    jar.enabled false
    shadowJar {
        configurations = [project.configurations.flinkCompile]
        classifier = null
    }

    tasks.addRule("Pattern: runFlink<ClassName>") { String taskName ->
        def taskPrefix = "runFlink"
        if (taskName.startsWith(taskPrefix)) {
            task(taskName) {
                doLast {
                    def jobName = taskName.replaceFirst(taskPrefix, "")
                    ssh.run {
                        session(remotes.hadoopMaster) {
                            def executable = "${project.name}-${version}.jar"
                            put from: "$project.buildDir/libs/$executable", into: '/vagrant/exchange'
                            execute "flink run -c net.fast2smart.streaming.${jobName} -d /vagrant/exchange/$executable"
                        }
                    }
                }
            }
        }
    }

    tasks.addRule("Pattern: stopFlink<ClassName>") { String taskName ->
        def taskPrefix = "stopFlink"
        if (taskName.startsWith(taskPrefix)) {
            task(taskName) {
                doLast {
                    def jobName = taskName.replaceFirst(taskPrefix, "")
                    ssh.run {
                        session(remotes.hadoopMaster) {
                            execute "flink cancel `flink list | grep $jobName | awk '{split(\$0,a,\" \"); print a[4]}'`"
                        }
                    }
                }
            }
        }
    }

    build.dependsOn shadowJar
}

project(':spark') {
    apply plugin: 'scala'
    apply plugin: 'com.github.johnrengelman.shadow'

    sourceSets {
        spark
        main {
            compileClasspath += configurations.sparkCompileClasspath
            runtimeClasspath += configurations.sparkRuntimeClasspath
        }
    }

    dependencies {
        //scala
        compile 'org.scala-lang:scala-library:2.11.+'

        //spark
        compile('org.apache.spark:spark-core_2.11:2.2.0') {
            exclude module: 'slf4j-log4j12'
        }
        compile('org.apache.spark:spark-sql_2.11:2.2.0') {
            exclude module: 'slf4j-log4j12'
        }
        compile('org.apache.spark:spark-streaming_2.11:2.2.0') {
            exclude module: 'slf4j-log4j12'
        }
        sparkCompile('org.apache.spark:spark-streaming-kafka-0-10_2.11:2.2.0') {
            exclude module: 'slf4j-log4j12'
        }
        //json
        sparkCompile 'org.json4s:json4s-native_2.11:3.2.11'
        sparkCompile 'org.json4s:json4s-jackson_2.11:3.2.11'
        //cassandra
        sparkCompile('com.datastax.spark:spark-cassandra-connector_2.11:2.0.5') {
            exclude module: 'slf4j-log4j12'
        }
        //database
        sparkCompile 'com.h2database:h2:1.4.193'
    }

    task aggregation(type:JavaExec) {
        main = 'net.fast2smart.batch.AccountAggregation'
        classpath = sourceSets.main.runtimeClasspath
    }

    tasks.addRule("Pattern: runSpark<ClassName>") { String taskName ->
        def taskPrefix = "runSpark"
        if (taskName.startsWith(taskPrefix)) {
            task(taskName) {
                doLast {
                    def jobName = taskName.replaceFirst(taskPrefix, "")
                    ssh.run {
                        session(remotes.hadoopMaster) {
                            def executable = "${project.name}-${version}.jar"
                            put from: "$project.buildDir/libs/$executable", into: '/vagrant/exchange'
                            execute "hdfs dfs -touchz /tmp/${jobName}.running"
                            execute "spark-submit --master yarn --class net.fast2smart.streaming.${jobName} --conf spark.yarn.submit.waitAppCompletion=false --deploy-mode cluster --executor-memory 1G --num-executors 3 /vagrant/exchange/$executable"
                        }
                    }
                }
            }
        }
    }

    tasks.addRule("Pattern: stopSpark<ClassName>") { String taskName ->
        def taskPrefix = "stopSpark"
        if (taskName.startsWith(taskPrefix)) {
            task(taskName) {
                doLast {
                    def jobName = taskName.replaceFirst(taskPrefix, "")
                    ssh.run {
                        session(remotes.hadoopMaster) {
                            execute "hdfs dfs -rm -f /tmp/${jobName}.running"
                        }
                    }
                }
            }
        }
    }

    jar.enabled false
    shadowJar {
        configurations = [project.configurations.sparkCompile]
        classifier = null
    }

    build.dependsOn shadowJar
}

project(':systemtest') {
    apply plugin: 'groovy'

    dependencies {
        //groovy
        compile 'org.codehaus.groovy:groovy-all:2.4.+'


        testCompile 'org.spockframework:spock-core:1.0-groovy-2.4'
        testCompile('com.athaydes:spock-reports:1.2.+' ) {
            transitive = false
        }
        testCompile 'org.codehaus.groovy.modules.http-builder:http-builder:0.7.1'
        testCompile 'com.github.zhicwu:cassandra-jdbc-driver:0.6.4'
        testCompile 'com.h2database:h2:1.4.193'
    }

    test {
        exclude '**/*Spec.class'
    }
}

task wrapper(type: Wrapper) {
    gradleVersion = '4.2'
    distributionUrl = 'https://services.gradle.org/distributions/gradle-4.2-all.zip'
}

idea {
    targetVersion = '17'
}