Skip to content

Commit

Permalink
Resolved #480
Browse files Browse the repository at this point in the history
  • Loading branch information
@sauterl
sauterl committed Jul 12, 2024
1 parent fd5cd6d commit 2802c5d
Show file tree
Hide file tree
Showing 4 changed files with 55 additions and 18 deletions.
6 changes: 3 additions & 3 deletions backend/src/main/kotlin/dev/dres/data/model/run/AbstractInteractiveTask.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ abstract class AbstractInteractiveTask(store: TransientEntityStore, task: DbTask
.asSequence().map {
ItemRange(it.item?.name!!, it.start!!, it.end!!)
}.toSet()
BasicJudgementValidator(template.toApi(), this.store, knownCorrectRanges = knownRanges)
BasicJudgementValidator(template.toApi(), this.store, template.taskGroup.type.toApi(), knownCorrectRanges = knownRanges)
}

DbTargetOption.VOTE -> {
Expand All @@ -62,12 +62,12 @@ abstract class AbstractInteractiveTask(store: TransientEntityStore, task: DbTask
val parameters =
template.taskGroup.type.configurations.filter { it.key eq targetOption.description }
.asSequence().associate { it.key to it.value }
BasicVoteValidator(template.toApi(), this.store, knownCorrectRanges = knownRanges, parameters = parameters)
BasicVoteValidator(template.toApi(), this.store, template.taskGroup.type.toApi(), knownCorrectRanges = knownRanges, parameters = parameters)
}

else -> throw IllegalStateException("The provided target option ${targetOption.description} is not supported by interactive tasks.")
}
}
}

}
}
4 changes: 2 additions & 2 deletions backend/src/main/kotlin/dev/dres/data/model/run/AbstractNonInteractiveTask.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,11 @@ abstract class AbstractNonInteractiveTask(store: TransientEntityStore, task: DbT
.asSequence().map {
ItemRange(it.item?.name!!, it.start!!, it.end!!)
}.toSet()
BasicJudgementValidator(template.toApi(), this.store, knownCorrectRanges = knownRanges)
BasicJudgementValidator(template.toApi(), this.store, template.taskGroup.type.toApi(), knownCorrectRanges = knownRanges)
}

else -> throw IllegalStateException("The provided target option ${targetOption.description} is not supported by interactive tasks.")
}
}
}
}
}
34 changes: 29 additions & 5 deletions backend/src/main/kotlin/dev/dres/run/validation/judged/BasicJudgementValidator.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package dev.dres.run.validation.judged
import dev.dres.api.rest.types.evaluation.submission.ApiAnswerSet
import dev.dres.api.rest.types.evaluation.submission.ApiVerdictStatus
import dev.dres.api.rest.types.template.tasks.ApiTaskTemplate
import dev.dres.api.rest.types.template.tasks.ApiTaskType
import dev.dres.data.model.submissions.*
import dev.dres.run.audit.AuditLogger
import dev.dres.run.exceptions.JudgementTimeoutException
Expand All @@ -22,15 +23,28 @@ import kotlin.concurrent.write
*
* @author Luca Rossetto
* @author Ralph Gasser
* @version 2.0.0
* @author Loris Sauter
*
* @version 2.1.0
*/
open class BasicJudgementValidator(override val taskTemplate: ApiTaskTemplate, protected val store: TransientEntityStore, knownCorrectRanges: Collection<ItemRange> = emptyList(), knownWrongRanges: Collection<ItemRange> = emptyList(),

open class BasicJudgementValidator(
override val taskTemplate: ApiTaskTemplate,
protected val store: TransientEntityStore,
private val taskType: ApiTaskType,
knownCorrectRanges: Collection<ItemRange> = emptyList(),
knownWrongRanges: Collection<ItemRange> = emptyList(),
) : AnswerSetValidator, JudgementValidator {

companion object {
private val counter = AtomicInteger()
private const val judgementTimeout = 60_000 //ms until a request is re-scheduled

/**
* The key used to read the task type configuration order
* The value expected under this key is LIFO, which results in LIFO ordering of the queue,
* all other parameters or the absence of one results in the default FIFO behaviour.
*/
private const val CONFIGURATION_ORDER_KEY = "JUDGEMENT.order"
}

/** The [BasicJudgementValidator]'s ID is simply an auto-incrementing number. */
Expand All @@ -43,7 +57,7 @@ open class BasicJudgementValidator(override val taskTemplate: ApiTaskTemplate, p
private val updateLock = ReentrantReadWriteLock()

/** Internal queue that keeps track of all the [AnswerSetId]s and associated [ItemRange]s that require judgement. */
private val queue: Queue<Pair<AnswerSetId,ItemRange>> = LinkedList()
private val queue: Deque<Pair<AnswerSetId,ItemRange>> = LinkedList()

/** Internal map of all [AnswerSetId]s and associated [ItemRange]s that have been retrieved by a judge and are pending a verdict. */
private val waiting = HashMap<String, Pair<AnswerSetId,ItemRange>>()
Expand All @@ -57,6 +71,12 @@ open class BasicJudgementValidator(override val taskTemplate: ApiTaskTemplate, p
/** Internal map of known [ItemRange]s to associated [DbVerdictStatus]. */
private val cache: MutableMap<ItemRange, DbVerdictStatus> = ConcurrentHashMap()

/** Internal flag whether to use LIFO */
private val lifo = when(this.taskType.configuration.getOrDefault(CONFIGURATION_ORDER_KEY, "fifo")) {
"lifo" -> true
else -> false
}

init {
knownCorrectRanges.forEach { this.cache[it] = DbVerdictStatus.CORRECT }
knownWrongRanges.forEach { this.cache[it] = DbVerdictStatus.WRONG }
Expand Down Expand Up @@ -99,7 +119,11 @@ open class BasicJudgementValidator(override val taskTemplate: ApiTaskTemplate, p
answerSet.status = cachedStatus
} else if (itemRange !in this.queuedItemRanges.keys) {
this.updateLock.write {
this.queue.offer(answerSet.id to itemRange)
if(this.lifo){
this.queue.offerFirst(answerSet.id to itemRange)
}else{
this.queue.offerLast(answerSet.id to itemRange)
}
this.queuedItemRanges[itemRange] = mutableListOf(answerSet.id)
}
} else {
Expand Down
29 changes: 21 additions & 8 deletions backend/src/main/kotlin/dev/dres/run/validation/judged/BasicVoteValidator.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package dev.dres.run.validation.judged
import dev.dres.api.rest.types.evaluation.submission.ApiAnswerSet
import dev.dres.api.rest.types.evaluation.submission.ApiVerdictStatus
import dev.dres.api.rest.types.template.tasks.ApiTaskTemplate
import dev.dres.api.rest.types.template.tasks.ApiTaskType
import dev.dres.data.model.submissions.*
import dev.dres.run.validation.interfaces.VoteValidator
import jetbrains.exodus.database.TransientEntityStore
Expand All @@ -19,15 +20,25 @@ import kotlin.concurrent.write
*
* @author Luca Rossetto
* @author Ralph Gasser
* @version 2.0.0
* @author Ralph Gasser
*
* @version 2.1.0
*/
class BasicVoteValidator(taskTemplate: ApiTaskTemplate, store: TransientEntityStore, knownCorrectRanges: Collection<ItemRange> = emptyList(), knownWrongRanges: Collection<ItemRange> = emptyList(), private val minimumVotes: Int = defaultMinimimVotes, private val voteDifference: Int = defaultVoteDifference) : BasicJudgementValidator(taskTemplate, store, knownCorrectRanges, knownWrongRanges), VoteValidator {

constructor(taskTemplate: ApiTaskTemplate, store: TransientEntityStore, knownCorrectRanges: Collection<ItemRange> = emptyList(), knownWrongRanges: Collection<ItemRange> = emptyList(), parameters: Map<String, String>): this(
taskTemplate, store,
class BasicVoteValidator(
taskTemplate: ApiTaskTemplate,
store: TransientEntityStore,
taskType: ApiTaskType,
knownCorrectRanges: Collection<ItemRange> = emptyList(),
knownWrongRanges: Collection<ItemRange> = emptyList(),
private val minimumVotes: Int = defaultMinimimVotes,
private val voteDifference: Int = defaultVoteDifference
) : BasicJudgementValidator(taskTemplate, store, taskType, knownCorrectRanges, knownWrongRanges), VoteValidator {

constructor(taskTemplate: ApiTaskTemplate, store: TransientEntityStore, taskType: ApiTaskType, knownCorrectRanges: Collection<ItemRange> = emptyList(), knownWrongRanges: Collection<ItemRange> = emptyList(), parameters: Map<String, String>): this(
taskTemplate, store, taskType,
knownCorrectRanges, knownWrongRanges,
parameters.getOrDefault("minimumVotes", "$defaultMinimimVotes").toIntOrNull() ?: defaultMinimimVotes,
parameters.getOrDefault("voteDifference", "$defaultVoteDifference").toIntOrNull() ?: defaultVoteDifference
parameters.getOrDefault(CONFIGURATION_MINIMUM_VOTES_KEY, "$defaultMinimimVotes").toIntOrNull() ?: defaultMinimimVotes,
parameters.getOrDefault(CONFIGURATION_VOTE_DIFF_KEY, "$defaultVoteDifference").toIntOrNull() ?: defaultVoteDifference
)

init {
Expand All @@ -37,6 +48,8 @@ class BasicVoteValidator(taskTemplate: ApiTaskTemplate, store: TransientEntitySt
companion object {
private val defaultMinimimVotes = 5
private val defaultVoteDifference = 1
private const val CONFIGURATION_MINIMUM_VOTES_KEY = "VOTE.minimumVotes"
private const val CONFIGURATION_VOTE_DIFF_KEY = "VOTE.voteDifference"
}

/** Internal queue of [AnswerSetId]s that pend voting. */
Expand Down Expand Up @@ -114,4 +127,4 @@ class BasicVoteValidator(taskTemplate: ApiTaskTemplate, store: TransientEntitySt
val others = sum - max
return max - others >= voteDifference
}
}
}

0 comments on commit 2802c5d

Please sign in to comment.