Skip to content

Commit

Permalink
Merge pull request #5 from RADAR-base/radarAuthJersey
Browse files Browse the repository at this point in the history 
Use radar-auth-jersey
  • Loading branch information
@blootsvoets
blootsvoets authored Oct 2, 2019
2 parents a8deacb + 51d3214 commit 6b66710
Show file tree
Hide file tree
Showing 40 changed files with 453 additions and 1,236 deletions.
9 changes: 6 additions & 3 deletions integration-test/build.gradle.kts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,19 @@ plugins {
}

project.extra.apply {
set("okhttpVersion", "4.0.1")
set("okhttpVersion", "4.2.0")
set("kafkaVersion", "2.3.0")
set("jacksonVersion", "2.9.9.1")
set("jacksonDataVersion", "2.9.9")
set("jacksonVersion", "2.9.10")
set("jacksonDataVersion", "2.9.10")
}

repositories {
jcenter()
mavenLocal()
maven(url = "http://packages.confluent.io/maven/")
maven(url = "https://dl.bintray.com/radar-cns/org.radarcns")
maven(url = "https://dl.bintray.com/radar-base/org.radarbase")
maven(url = "https://repo.thehyve.nl/content/repositories/snapshots")
maven(url = "http://oss.jfrog.org/artifactory/oss-snapshot-local/")
}

Expand Down
12 changes: 4 additions & 8 deletions ...ration-test/src/integrationTest/java/org/radarbase/connect/upload/UploadSourceTaskTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,9 +60,10 @@ class UploadSourceTaskTest {
"upload.source.client.tokenUrl" to tokenUrl,
"upload.source.backend.baseUrl" to baseUri,
"upload.source.poll.interval.ms" to "10000",
"upload.source.record.converter.classes" to
"org.radarbase.connect.upload.converter.AccelerometerCsvRecordConverter,org.radarbase.connect.upload.converter.altoida.AltoidaZipFileRecordConverter"

"upload.source.record.converter.classes" to listOf(
"org.radarbase.connect.upload.converter.AccelerometerCsvRecordConverter",
"org.radarbase.connect.upload.converter.altoida.AltoidaZipFileRecordConverter"
).joinToString(separator=",")
)

sourceTask.start(settings)
Expand Down Expand Up @@ -102,7 +103,6 @@ class UploadSourceTaskTest {
@Test
@DisplayName("Records of no registered converters should not be polled")
fun noConverterFound() {

val sourceType = "acceleration-zip"
val fileName = "TEST_ACC.zip"
val createdRecord = createRecordAndUploadContent(accessToken, sourceType, fileName)
Expand All @@ -116,17 +116,14 @@ class UploadSourceTaskTest {
val metadata = retrieveRecordMetadata(accessToken, createdRecord.id!!)
assertNotNull(metadata)
assertEquals("READY", metadata.status)

}

@Test
@DisplayName("Should mark FAILED if the record data does not match the source-type")
fun incorrectSourceTypeForRecord() {

val sourceType = "phone-acceleration"
val fileName = "TEST_ACC.zip"
val createdRecord = createRecordAndUploadContent(accessToken, sourceType, fileName)
assertNotNull(createdRecord)
assertNotNull(createdRecord.id)

val sourceRecords = sourceTask.poll()
Expand All @@ -136,6 +133,5 @@ class UploadSourceTaskTest {
val metadata = retrieveRecordMetadata(accessToken, createdRecord.id!!)
assertNotNull(metadata)
assertEquals("FAILED", metadata.status)

}
}
8 changes: 3 additions & 5 deletions ...tegrationTest/java/org/radarbase/connect/upload/api/UploadBackendClientIntegrationTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,6 @@ import java.io.File

@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class UploadBackendClientIntegrationTest {


private lateinit var uploadBackendClient: UploadBackendClient

private lateinit var logRepository: LogRepository
Expand Down Expand Up @@ -113,7 +111,7 @@ class UploadBackendClientIntegrationTest {
val converter = AccelerometerCsvRecordConverter()
converter.initialize(sourceType, uploadBackendClient, logRepository, emptyMap())

val recordToProcess = records.records.filter { recordDTO -> recordDTO.sourceType == sourceTypeName }.first()
val recordToProcess = records.records.first { recordDTO -> recordDTO.sourceType == sourceTypeName }
createdRecord.metadata = uploadBackendClient.updateStatus(recordToProcess.id!!, recordToProcess.metadata!!.copy(status = "PROCESSING", message = "The record is being processed"))
val convertedRecords = converter.convert(records.records.first())
assertNotNull(convertedRecords)
Expand All @@ -137,9 +135,9 @@ class UploadBackendClientIntegrationTest {
}

private fun retrieveFile(recordId: RecordDTO) {
uploadBackendClient.retrieveFile(recordId, fileName).use { response ->
uploadBackendClient.retrieveFile(recordId, fileName) { response ->
assertNotNull(response)
val responseData = response!!.bytes()
val responseData = response.bytes()
assertThat(responseData.size.toLong(), equalTo(File(fileName).length()))
assertThat(responseData, equalTo(File(fileName).readBytes()))
}
Expand Down
162 changes: 75 additions & 87 deletions integration-test/src/integrationTest/java/org/radarbase/connect/upload/util/TestBase.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,9 @@ import okhttp3.RequestBody.Companion.toRequestBody
import org.hamcrest.CoreMatchers
import org.hamcrest.MatcherAssert.assertThat
import org.hamcrest.Matchers
import org.hamcrest.Matchers.*
import org.junit.jupiter.api.Assertions
import org.junit.jupiter.api.Assertions.assertTrue
import org.radarbase.connect.upload.api.*
import org.radarbase.connect.upload.auth.ClientCredentialsAuthorizer
import org.radarbase.upload.Config
Expand Down Expand Up @@ -64,22 +66,22 @@ class TestBase {

const val uploadConnectSecret = "upload_secret"

const val BEARER = "Bearer "
private const val BEARER = "Bearer "

const val USER = "sub-1"
private const val USER = "sub-1"

const val PROJECT = "radar"
private const val PROJECT = "radar"

const val SOURCE = "03d28e5c-e005-46d4-a9b3-279c27fbbc83"
private const val SOURCE = "03d28e5c-e005-46d4-a9b3-279c27fbbc83"

val APPLICATION_JSON = "application/json; charset=utf-8".toMediaType()
private val APPLICATION_JSON = "application/json; charset=utf-8".toMediaType()

val TEXT_CSV = "text/csv; charset=utf-8".toMediaType()
private val TEXT_CSV = "text/csv; charset=utf-8".toMediaType()

val httpClient = OkHttpClient()


val sourceType = SourceTypeDTO(
private val sourceType = SourceTypeDTO(
name = sourceTypeName,
topics = mutableSetOf("test_topic"),
contentTypes = mutableSetOf("application/text"),
Expand All @@ -88,26 +90,28 @@ class TestBase {
configuration = mutableMapOf("setting1" to "value1", "setting2" to "value2")
)

val altoidaZip = SourceTypeDTO(
private val altoidaZip = SourceTypeDTO(
name = "altoida-zip",
topics = mutableSetOf("test_topic"),
contentTypes = mutableSetOf("application/zip"),
timeRequired = false,
sourceIdRequired = false,
configuration = emptyMap()
configuration = mutableMapOf()
)

val accelerationZip = SourceTypeDTO(
private val accelerationZip = SourceTypeDTO(
name = "acceleration-zip",
topics = mutableSetOf("test_topic_Acc"),
contentTypes = mutableSetOf("application/zip"),
timeRequired = false,
sourceIdRequired = false,
configuration = emptyMap()
configuration = mutableMapOf()
)

val uploadBackendConfig = Config(
managementPortalUrl = "http://localhost:8090/managementportal",
clientId = "radar_upload_backend",
clientSecret = "secret",
baseUri = URI.create(baseUri),
jdbcDriver = "org.postgresql.Driver",
jdbcUrl = "jdbc:postgresql://localhost:5434/uploadconnector",
Expand All @@ -116,7 +120,7 @@ class TestBase {
sourceTypes = listOf(sourceType, altoidaZip, accelerationZip)
)

val mapper = ObjectMapper(JsonFactory())
private val mapper: ObjectMapper = ObjectMapper(JsonFactory())
.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
.registerModule(KotlinModule())
.registerModule(JavaTimeModule())
Expand All @@ -126,10 +130,9 @@ class TestBase {
httpClient,
uploadConnectClient,
uploadConnectSecret,
tokenUrl
)
tokenUrl)

fun call(
private fun call(
httpClient: OkHttpClient,
expectedStatus: Int,
requestSupplier: (Request.Builder) -> Request.Builder
Expand All @@ -142,56 +145,62 @@ class TestBase {
println(tree)
tree
}
assertThat(response.code, CoreMatchers.`is`(expectedStatus))
assertThat(response.code, `is`(expectedStatus))
body
}
}

fun Any.toJsonString(): String = mapper.writeValueAsString(this)
private fun<T> call(
httpClient: OkHttpClient,
expectedStatus: Int,
parseClass: Class<T>,
requestSupplier: Request.Builder.() -> Request.Builder
): T {
val request = requestSupplier(Request.Builder()).build()
println(request.url)
return httpClient.newCall(request).execute().use { response ->
assertThat(response.code, `is`(expectedStatus))
assertThat(response.body, not(nullValue()))
mapper.readValue(response.body?.byteStream(), parseClass)
.also { assertThat(it, not(nullValue())) }
.also { println(it!!.toJsonString()) }
}
}

private fun Any.toJsonString(): String = mapper.writeValueAsString(this)

fun call(
private fun call(
httpClient: OkHttpClient,
expectedStatus: Response.Status,
requestSupplier: (Request.Builder) -> Request.Builder
): JsonNode? {
return call(httpClient, expectedStatus.statusCode, requestSupplier)
}
): JsonNode? = call(httpClient, expectedStatus.statusCode, requestSupplier)

fun call(
private fun call(
httpClient: OkHttpClient,
expectedStatus: Response.Status,
stringProperty: String,
requestSupplier: (Request.Builder) -> Request.Builder
): String {
return call(httpClient, expectedStatus, requestSupplier)?.get(stringProperty)?.asText()
?: throw AssertionError("String property $stringProperty not found")
}
requestSupplier: Request.Builder.() -> Request.Builder
): String = call(httpClient, expectedStatus, requestSupplier)
?.get(stringProperty)
?.asText()
?: throw AssertionError("String property $stringProperty not found")

fun retrieveRecordMetadata(accessToken: String, recordId: Long): RecordMetadataDTO {

val requestToUploadFile = Request.Builder()
.url("$baseUri/records/$recordId/metadata")
.get()
.addHeader("Authorization", BEARER + accessToken)
.build()
val response = httpClient.newCall(requestToUploadFile).execute()
Assertions.assertTrue(response.isSuccessful)

return mapper.readValue(response.body?.string(), RecordMetadataDTO::class.java)
return call(httpClient,200, RecordMetadataDTO::class.java) {
url("$baseUri/records/$recordId/metadata")
addHeader("Authorization", BEARER + accessToken)
}
}

fun getAccessToken() : String {
return call(httpClient, Response.Status.OK, "access_token") {
it.url(tokenUrl)
.addHeader("Authorization", Credentials.basic("radar_upload_test_client", "test"))
.post(FormBody.Builder()
.add("grant_type", "client_credentials")
.build())
}
fun getAccessToken() : String = call(httpClient, Response.Status.OK, "access_token") {
url(tokenUrl)
addHeader("Authorization", Credentials.basic("radar_upload_test_client", "test"))
post(FormBody.Builder()
.add("grant_type", "client_credentials")
.build())
}

fun createRecordAndUploadContent(accessToken: String, sourceType: String, fileName: String): RecordDTO {

val record = RecordDTO(
id = null,
data = RecordDataDTO(
Expand All @@ -204,20 +213,14 @@ class TestBase {
metadata = null
)

val request = Request.Builder()
.url("$baseUri/records")
.post(record.toJsonString().toRequestBody(APPLICATION_JSON))
.addHeader("Authorization", BEARER + accessToken)
.addHeader("Content-type", "application/json")
.build()

val response = httpClient.newCall(request).execute()
Assertions.assertTrue(response.isSuccessful)

val recordCreated = mapper.readValue(response.body?.string(), RecordDTO::class.java)
Assertions.assertNotNull(recordCreated)
Assertions.assertNotNull(recordCreated.id)
assertThat(recordCreated?.id!!, Matchers.greaterThan(0L))
val recordCreated = call(httpClient, 201, RecordDTO::class.java) {
url("$baseUri/records")
post(record.toJsonString().toRequestBody(APPLICATION_JSON))
addHeader("Authorization", BEARER + accessToken)
addHeader("Content-type", "application/json")
}
assertThat(recordCreated.id, not(nullValue()))
assertThat(recordCreated.id!!, greaterThan(0L))

//Test uploading request contentFile for created record
uploadContent(recordCreated.id!!, fileName, accessToken)
Expand All @@ -229,36 +232,21 @@ class TestBase {
//Test uploading request contentFile
val file = File(fileName)

val requestToUploadFile = Request.Builder()
.url("$baseUri/records/$recordId/contents/$fileName")
.put(file.asRequestBody(TEXT_CSV))
.addHeader("Authorization", BEARER + clientUserToken)
.build()

val uploadResponse = httpClient.newCall(requestToUploadFile).execute()
Assertions.assertTrue(uploadResponse.isSuccessful)

val content = mapper.readValue(uploadResponse.body?.string(), ContentsDTO::class.java)
Assertions.assertNotNull(content)
Assertions.assertEquals(fileName, content.fileName)
val content = call(httpClient, 201, ContentsDTO::class.java) {
url("$baseUri/records/$recordId/contents/$fileName")
put(file.asRequestBody(TEXT_CSV))
addHeader("Authorization", BEARER + clientUserToken)
}
assertThat(content.fileName, equalTo(fileName))
}


private fun markReady(recordId: Long, clientUserToken: String) {
//Test marking record READY
val requestToUploadFile = Request.Builder()
.url("$baseUri/records/$recordId/metadata")
.post("{\"status\":\"READY\",\"revision\":1}".toRequestBody("application/json".toMediaType()))
.addHeader("Authorization", BEARER + clientUserToken)
.build()

val uploadResponse = httpClient.newCall(requestToUploadFile).execute()
Assertions.assertTrue(uploadResponse.isSuccessful)

val metadata = mapper.readValue(uploadResponse.body?.string(), RecordMetadataDTO::class.java)
Assertions.assertNotNull(metadata)
Assertions.assertEquals("READY", metadata.status)
val metadata = call(httpClient, 200, RecordMetadataDTO::class.java) {
url("$baseUri/records/$recordId/metadata")
post("{\"status\":\"READY\",\"revision\":1}".toRequestBody("application/json".toMediaType()))
addHeader("Authorization", BEARER + clientUserToken)
}
assertThat(metadata.status, equalTo("READY"))
}

}
}
8 changes: 3 additions & 5 deletions kafka-connect-upload-source/build.gradle.kts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@ plugins {

project.extra.apply {
set("kafkaVersion", "2.3.0")
set("okhttpVersion", "3.14.2")
set("jacksonVersion", "2.9.9.1")
set("jacksonDataVersion", "2.9.9")
set("okhttpVersion", "4.2.0")
set("jacksonVersion", "2.9.10")
set("jacksonDataVersion", "2.9.10")
set("openCsvVersion", "4.6")
set("confluentVersion", "5.3.0")
set("radarSchemaVersion", "0.5.2-SNAPSHOT")
Expand Down Expand Up @@ -44,12 +44,10 @@ dependencies {
implementation("com.fasterxml.jackson.module:jackson-module-kotlin:${project.extra["jacksonDataVersion"]}")
implementation("com.opencsv:opencsv:${project.extra["openCsvVersion"]}")


// Included in connector runtime
compileOnly("org.apache.kafka:connect-api:${project.extra["kafkaVersion"]}")
implementation(kotlin("stdlib-jdk8"))


testImplementation("org.junit.jupiter:junit-jupiter:5.4.2")
testImplementation("org.hamcrest:hamcrest-all:1.3")
testImplementation("org.apache.kafka:connect-api:${project.extra["kafkaVersion"]}")
Expand Down
Loading

0 comments on commit 6b66710

Please sign in to comment.