Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Inclusion of the DoF drift detection method #276

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Inclusion of the DoF drift detection method #276

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions .project
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,15 @@
<natures>
<nature>org.eclipse.m2e.core.maven2Nature</nature>
</natures>
<filteredResources>
<filter>
<id>1680708499525</id>
<name></name>
<type>30</type>
<matcher>
<id>org.eclipse.core.resources.regexFilterMatcher</id>
<arguments>node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__</arguments>
</matcher>
</filter>
</filteredResources>
</projectDescription>
20 changes: 18 additions & 2 deletions moa/.classpath
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,8 @@
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-10">
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
<attributes>
<attribute name="module" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
Expand All @@ -35,5 +34,22 @@
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="src" path="target/generated-sources/annotations">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
<attribute name="ignore_optional_problems" value="true"/>
<attribute name="m2e-apt" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="target/test-classes" path="target/generated-test-sources/test-annotations">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
<attribute name="ignore_optional_problems" value="true"/>
<attribute name="m2e-apt" value="true"/>
<attribute name="test" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="output" path="target/classes"/>
</classpath>
11 changes: 11 additions & 0 deletions moa/.project
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,4 +20,15 @@
<nature>org.eclipse.jdt.core.javanature</nature>
<nature>org.eclipse.m2e.core.maven2Nature</nature>
</natures>
<filteredResources>
<filter>
<id>1680708499504</id>
<name></name>
<type>30</type>
<matcher>
<id>org.eclipse.core.resources.regexFilterMatcher</id>
<arguments>node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__</arguments>
</matcher>
</filter>
</filteredResources>
</projectDescription>
2 changes: 2 additions & 0 deletions moa/.settings/org.eclipse.jdt.apt.core.prefs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
eclipse.preferences.version=1
org.eclipse.jdt.apt.aptEnabled=false
11 changes: 7 additions & 4 deletions moa/.settings/org.eclipse.jdt.core.prefs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,16 @@
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
org.eclipse.jdt.core.compiler.codegen.targetPlatform=10
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
org.eclipse.jdt.core.compiler.compliance=10
org.eclipse.jdt.core.compiler.compliance=1.8
org.eclipse.jdt.core.compiler.debug.lineNumber=generate
org.eclipse.jdt.core.compiler.debug.localVariable=generate
org.eclipse.jdt.core.compiler.debug.sourceFile=generate
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.release=enabled
org.eclipse.jdt.core.compiler.source=10
org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore
org.eclipse.jdt.core.compiler.processAnnotations=disabled
org.eclipse.jdt.core.compiler.release=disabled
org.eclipse.jdt.core.compiler.source=1.8
244 changes: 244 additions & 0 deletions moa/src/main/java/moa/classifiers/core/driftdetection/DoF.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,244 @@
package moa.classifiers.core.driftdetection;

import com.github.javacliparser.FloatOption;
import com.github.javacliparser.IntOption;
import com.yahoo.labs.samoa.instances.Instance;
import java.util.Arrays;
import moa.classifiers.AbstractClassifier;
import moa.classifiers.Classifier;
import moa.core.Measurement;
import moa.options.ClassOption;
import java.util.LinkedList;
import java.util.List;

/**
*
* <p>Parinaz Sobhani and Hamid Beigy: New Drift Detection Method for Data
* Streams. Adaptive and Intelligent Systems 2011: Volume 6943, pp 88-97.</p>
*
* Adapted to change classifier using threshold.
*
* @author Silas Garrido ([email protected])
*
*/
public class DoF extends AbstractClassifier {

public ClassOption learnerOption = new ClassOption("Learner", 'l',
"Classifier to train.", Classifier.class, "bayes.NaiveBayes");
public IntOption windowSizeOption = new IntOption(
"windowSize", 'w', "Window size.", 15, 0, Integer.MAX_VALUE);
public FloatOption thresholdOption = new FloatOption("stddev",
's', "Standard deviations.", 2.0);
protected Classifier classifier;
protected LinkedList<Instance> currentBatch;
protected LinkedList<Instance> referenceBatch;
protected double[] minAttributes;
protected double[] maxAttributes;
protected double[] knnWeight;
protected int[] knnIndex;
protected int countBatch;
protected boolean newReferenceBatch;
protected boolean initializeRange;
protected int changeDetected = 0;
private RunningStat rs;

private void initialize() {
this.currentBatch = new LinkedList();
this.referenceBatch = new LinkedList();
this.knnWeight = new double[this.windowSizeOption.getValue()];
this.knnIndex = new int[this.windowSizeOption.getValue()];
this.countBatch = 0;
this.newReferenceBatch = true;
this.initializeRange = true;
this.rs = new RunningStat();
}

@Override
public void resetLearningImpl() {
this.classifier = ((Classifier) getPreparedClassOption(this.learnerOption)).copy();
this.classifier.resetLearning();
initialize();
}

private void initializeRangeAttributes(Instance inst) {
this.minAttributes = new double[inst.numAttributes() - 1];
this.maxAttributes = new double[inst.numAttributes() - 1];
Arrays.fill(this.minAttributes, Double.MAX_VALUE);
Arrays.fill(this.maxAttributes, Double.MIN_VALUE);
}

private void calcRangeAttributes(Instance inst) {
double[] temp = inst.toDoubleArray();

for (int i = 0; i < this.minAttributes.length; i++) {
if ((inst.attribute(i).isNumeric()) && (temp[i] < this.minAttributes[i])) {
this.minAttributes[i] = temp[i];
}

if ((inst.attribute(i).isNumeric()) && (temp[i] > this.maxAttributes[i])) {
this.maxAttributes[i] = temp[i];
}
}
}

private double calcDist(Instance inst1, Instance inst2) {
double[] temp1 = inst1.toDoubleArray();
double[] temp2 = inst2.toDoubleArray();
double dist = 0.0, tempDist = 0.0;
int numAttributes = inst1.numAttributes() - 1;

for (int i = 0; i < numAttributes; i++) {
if (inst1.attribute(i).isNumeric()) {
tempDist = ((temp1[i] - temp2[i]) / (this.maxAttributes[i] - this.minAttributes[i]));
} else {
if (temp1[i] != temp2[i]) {
tempDist = 1.0;
}
}

tempDist = Math.pow(tempDist, 2.0);
dist += tempDist;
}

return Math.sqrt(dist);
}

private void nearestNeighbor() {
Arrays.fill(this.knnWeight, Double.MAX_VALUE);
double tempDist;

for (int i = 0; i < this.currentBatch.size(); i++) {
for (int j = 0; j < this.referenceBatch.size(); j++) {
tempDist = calcDist(this.currentBatch.get(i), this.referenceBatch.get(j));

if (tempDist < this.knnWeight[i]) {
this.knnWeight[i] = tempDist;
this.knnIndex[i] = j;
}
}
}
}

private double degreeOfDrift() {
double numerator = 0.0, denominator = 0.0;
double[] tempCurrent, tempReference;
int position;

for (int i = 0; i < this.knnWeight.length; i++) {
tempCurrent = this.currentBatch.get(i).toDoubleArray();
tempReference = this.referenceBatch.get(this.knnIndex[i]).toDoubleArray();
position = this.currentBatch.get(i).numAttributes() - 1;

if (tempCurrent[position] != tempReference[position]) {
numerator += ((this.knnWeight[i] == 0.0) ? 0.0 : (1.0 / this.knnWeight[i]));
}

denominator += ((this.knnWeight[i] == 0.0) ? 0.0 : (1.0 / this.knnWeight[i]));
}

return numerator / denominator;
}

@Override
public void trainOnInstanceImpl(Instance inst) {
if (this.initializeRange) {
initializeRangeAttributes(inst);
this.initializeRange = false;
}

calcRangeAttributes(inst);

if (this.newReferenceBatch) {
this.referenceBatch.addLast(inst);

if (this.countBatch + 1 == this.windowSizeOption.getValue()) {
this.countBatch = -1;
this.newReferenceBatch = false;
}
} else {
this.currentBatch.addLast(inst);

if (this.countBatch + 1 == this.windowSizeOption.getValue()) {
nearestNeighbor();

double tempDof = degreeOfDrift();
this.rs.put(tempDof);

LinkedList<Instance> temp = this.referenceBatch;
if (tempDof > this.rs.getAverage() + this.rs.getStandardDeviation() * this.thresholdOption.getValue() ||
tempDof < this.rs.getAverage() - this.rs.getStandardDeviation() * this.thresholdOption.getValue()) {
this.changeDetected++;
this.classifier.resetLearning();
initialize();
this.referenceBatch = temp;
this.newReferenceBatch = false;
}

this.countBatch = -1;
this.currentBatch.clear();
}
}

this.classifier.trainOnInstance(inst);
this.countBatch++;
}

@Override
public double[] getVotesForInstance(Instance inst) {
return this.classifier.getVotesForInstance(inst);
}

@Override
public boolean isRandomizable() {
return false;
}

@Override
public void getModelDescription(StringBuilder out, int indent) {
((AbstractClassifier) this.classifier).getModelDescription(out, indent);
}

@Override
protected Measurement[] getModelMeasurementsImpl() {
List<Measurement> measurementList = new LinkedList();
measurementList.add(new Measurement("Change detected", this.changeDetected));
Measurement[] modelMeasurements = ((AbstractClassifier) this.classifier).getModelMeasurements();
if (modelMeasurements != null) {
measurementList.addAll(Arrays.asList(modelMeasurements));
}
this.changeDetected = 0;
return measurementList.toArray(new Measurement[measurementList.size()]);
}
}

class RunningStat {

private int count = 0;
private double average = 0.0;
private double pwrSumAvg = 0.0;
private double stdDev = 0.0;

/**
* Incoming new values used to calculate the running statistics
*
* @param value
*/
public void put(double value) {

count++;
average += (value - average) / count;
pwrSumAvg += (value * value - pwrSumAvg) / count;
stdDev = Math.sqrt((pwrSumAvg * count - count * average * average) / (count - 1));

}

public double getAverage() {

return average;
}

public double getStandardDeviation() {

return Double.isNaN(stdDev) ? 0.0 : stdDev;
}
}
19 changes: 18 additions & 1 deletion weka-package/.classpath
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
<attribute name="test" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-11">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
Expand All @@ -23,5 +23,22 @@
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="src" path="target/generated-sources/annotations">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
<attribute name="ignore_optional_problems" value="true"/>
<attribute name="m2e-apt" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="target/test-classes" path="target/generated-test-sources/test-annotations">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
<attribute name="ignore_optional_problems" value="true"/>
<attribute name="m2e-apt" value="true"/>
<attribute name="test" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="output" path="target/classes"/>
</classpath>
11 changes: 11 additions & 0 deletions weka-package/.project
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,4 +20,15 @@
<nature>org.eclipse.jdt.core.javanature</nature>
<nature>org.eclipse.m2e.core.maven2Nature</nature>
</natures>
<filteredResources>
<filter>
<id>1680708499531</id>
<name></name>
<type>30</type>
<matcher>
<id>org.eclipse.core.resources.regexFilterMatcher</id>
<arguments>node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__</arguments>
</matcher>
</filter>
</filteredResources>
</projectDescription>
2 changes: 2 additions & 0 deletions weka-package/.settings/org.eclipse.jdt.apt.core.prefs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
eclipse.preferences.version=1
org.eclipse.jdt.apt.aptEnabled=false
11 changes: 7 additions & 4 deletions weka-package/.settings/org.eclipse.jdt.core.prefs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
org.eclipse.jdt.core.compiler.compliance=1.8
org.eclipse.jdt.core.compiler.codegen.targetPlatform=11
org.eclipse.jdt.core.compiler.compliance=11
org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.release=enabled
org.eclipse.jdt.core.compiler.source=1.8
org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore
org.eclipse.jdt.core.compiler.processAnnotations=disabled
org.eclipse.jdt.core.compiler.release=disabled
org.eclipse.jdt.core.compiler.source=11