diff --git a/project/build.scala b/project/build.scala index d6a6ff394..9f16f24d8 100644 --- a/project/build.scala +++ b/project/build.scala @@ -52,49 +52,23 @@ object build extends Build { id = "scoobi", base = file("."), configurations = Configurations.default ++ Seq(repl.Repl), - settings = Seq(name := "scoobi-parent") ++ - moduleSettings ++ - siteSettings ++ - releaseSettings ++ - Seq(sourceDirectory := file("ignoreme")) - ).aggregate(core, thrift) - - lazy val moduleSettings: Seq[Settings] = - Defaults.defaultSettings ++ - scoobiSettings ++ - buildSettings ++ - compilationSettings ++ - testingSettings ++ - publicationSettings ++ - mimaSettings ++ - notificationSettings ++ - dependencies.resolversSettings ++ - repl.settings - - lazy val core = Project( - id = "core", - base = file("scoobi-core"), - settings = moduleSettings ++ Seq( - name := "scoobi" - // We're playing games with SBT to avoid having to move everything from 'src' to 'scoobi-core' - , sourceDirectory := baseDirectory.value.getParentFile / "src" - , dependencies.dependencies - , (sourceGenerators in Compile) <+= (sourceManaged in Compile) map GenWireFormat.gen - ) + settings = Defaults.defaultSettings ++ + scoobiSettings ++ + dependencies.settings ++ + buildSettings ++ + compilationSettings ++ + testingSettings ++ + siteSettings ++ + publicationSettings ++ + mimaSettings ++ + notificationSettings ++ + releaseSettings ++ + repl.settings ) - lazy val thrift = Project( - id = "thrift" - , base = file("scoobi-thrift") - , settings = moduleSettings ++ Seq[Settings]( - name := "scoobi-thrift" - , libraryDependencies ++= dependencies.thrift ++ dependencies.specs2() - ) - ) - .dependsOn(core) - lazy val scoobiVersion = SettingKey[String]("scoobi-version", "defines the current Scoobi version") lazy val scoobiSettings: Seq[Settings] = Seq( + name := "scoobi", organization := "com.nicta", scoobiVersion in GlobalScope <<= version, scalaVersion := "2.11.2", @@ -115,6 +89,7 @@ object build extends Build { lazy val compilationSettings: Seq[Settings] = Seq( + (sourceGenerators in Compile) <+= (sourceManaged in Compile) map GenWireFormat.gen, incOptions := incOptions.value.withNameHashing(true), scalacOptions ++= Seq("-deprecation", "-unchecked", "-feature", "-language:_"), scalacOptions in Test ++= Seq("-Yrangepos") diff --git a/project/dependencies.scala b/project/dependencies.scala index eba60efcd..10c51612d 100644 --- a/project/dependencies.scala +++ b/project/dependencies.scala @@ -18,6 +18,7 @@ import Keys._ object dependencies { + lazy val settings = dependencies ++ resolversSettings lazy val dependencies = libraryDependencies ++= scoobi(scalaVersion.value) ++ hadoop(version.value) ++ @@ -61,10 +62,6 @@ object dependencies { "org.specs2" %% "specs2-html" % specs2Version , "org.specs2" %% "specs2-analysis" % specs2Version ).map(_ % "test") - val thrift = Seq( - "org.apache.thrift" % "libthrift" % "0.9.1" - ) - def repl(scalaVersion: String) = Seq( if (scalaVersion.contains("2.10")) "org.scala-lang" % "jline" % scalaVersion diff --git a/project/plugins.sbt b/project/plugins.sbt index f1ac11e26..bf3a37445 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -16,6 +16,4 @@ addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "0.1.6") addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "0.2.1") -addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.3.1") - -resolvers += Resolver.sonatypeRepo("releases") +addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.3.1") \ No newline at end of file diff --git a/scoobi-thrift/README.md b/scoobi-thrift/README.md deleted file mode 100644 index 8595d8d39..000000000 --- a/scoobi-thrift/README.md +++ /dev/null @@ -1,9 +0,0 @@ -Scoobi Thrift -============= - -An implementation of `WireFormat` and `SeqSchema` for the [Java implementation](https://thrift.apache.org/tutorial/java) -of [Apache Thrift](https://thrift.apache.org). - -```scala -import com.nicta.scoobi.io.thrift._ -``` \ No newline at end of file diff --git a/scoobi-thrift/src/main/scala/com/nicta/scoobi/io/thrift/ThriftSchema.scala b/scoobi-thrift/src/main/scala/com/nicta/scoobi/io/thrift/ThriftSchema.scala deleted file mode 100644 index 425feac1b..000000000 --- a/scoobi-thrift/src/main/scala/com/nicta/scoobi/io/thrift/ThriftSchema.scala +++ /dev/null @@ -1,48 +0,0 @@ -package com.nicta.scoobi.io.thrift - -import java.io.{DataInput, DataOutput} - -import com.nicta.scoobi.Scoobi._ -import org.apache.hadoop.io.BytesWritable - -/** - * Schema for creating Thrift WireFormat and SeqSchema instances. - */ -object ThriftSchema { - - /* WARNING THIS MUST BE A DEF OR OR IT CAN TRIGGER CONCURRENCY ISSUES WITH SHARED THRIFT SERIALIZERS */ - def mkThriftFmt[A](implicit m: Manifest[A], ev: A <:< ThriftLike): WireFormat[A] = new WireFormat[A] { - val serialiser = ThriftSerialiser() - // Call once when the implicit is created to avoid further reflection - val empty = m.runtimeClass.newInstance().asInstanceOf[A] - - def toWire(x: A, out: DataOutput) = { - val bytes = serialiser.toBytes(x) - out.writeInt(bytes.length) - out.write(bytes) - } - - def fromWire(in: DataInput): A = { - val size = in.readInt() - val bytes = new Array[Byte](size) - in.readFully(bytes) - serialiser.fromBytes(empty, bytes) - } - - override def toString = "ThriftObject" - } - - /* WARNING THIS MUST BE A DEF OR OR IT CAN TRIGGER CONCURRENCY ISSUES WITH SHARED THRIFT SERIALIZERS*/ - def mkThriftSchema[A](implicit m: Manifest[A], ev: A <:< ThriftLike) = new SeqSchema[A] { - type SeqType = BytesWritable - val serialiser = ThriftSerialiser() - // Call once when the implicit is created to avoid further reflection - val empty = m.runtimeClass.newInstance().asInstanceOf[A] - - def toWritable(x: A) = new BytesWritable(serialiser.toBytes(x)) - - def fromWritable(x: BytesWritable): A = serialiser.fromBytes(empty, x.getBytes) - - val mf: Manifest[SeqType] = implicitly - } -} diff --git a/scoobi-thrift/src/main/scala/com/nicta/scoobi/io/thrift/ThriftSerialiser.scala b/scoobi-thrift/src/main/scala/com/nicta/scoobi/io/thrift/ThriftSerialiser.scala deleted file mode 100644 index c96fcc9b9..000000000 --- a/scoobi-thrift/src/main/scala/com/nicta/scoobi/io/thrift/ThriftSerialiser.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.nicta.scoobi.io.thrift - -import org.apache.thrift.{TDeserializer, TSerializer} -import org.apache.thrift.protocol.TCompactProtocol - -/** - * Util for converting a `ThriftLike` object to and from bytes. - * - * WARNING: This class is _not_ threadsafe and should be used with extreme caution! - * - * https://issues.apache.org/jira/browse/THRIFT-2218 - */ -case class ThriftSerialiser() { - - val serialiser = new TSerializer(new TCompactProtocol.Factory) - val deserialiser = new TDeserializer(new TCompactProtocol.Factory) - - def toBytes[A](a: A)(implicit ev: A <:< ThriftLike): Array[Byte] = - serialiser.serialize(ev(a)) - - def fromBytes[A](empty: A, bytes: Array[Byte])(implicit ev: A <:< ThriftLike): A = { - val e = ev(empty).deepCopy - e.clear() - deserialiser.deserialize(e, bytes) - e.asInstanceOf[A] - } -} diff --git a/scoobi-thrift/src/main/scala/com/nicta/scoobi/io/thrift/package.scala b/scoobi-thrift/src/main/scala/com/nicta/scoobi/io/thrift/package.scala deleted file mode 100644 index b25c2e7b0..000000000 --- a/scoobi-thrift/src/main/scala/com/nicta/scoobi/io/thrift/package.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.nicta.scoobi.io - -import com.nicta.scoobi.Scoobi._ - -package object thrift { - - type ThriftLike = org.apache.thrift.TBase[_ <: org.apache.thrift.TBase[_, _], _ <: org.apache.thrift.TFieldIdEnum] - - implicit def ThriftWireFormat[A](implicit m: Manifest[A], ev: A <:< ThriftLike): WireFormat[A] =ThriftSchema.mkThriftFmt[A] - - implicit def ThriftSeqSchema[A](implicit m: Manifest[A], ev: A <:< ThriftLike): SeqSchema[A] = ThriftSchema.mkThriftSchema[A] -} diff --git a/scoobi-thrift/src/test/java/com/nicta/scoobi/io/thrift/MyThrift.java b/scoobi-thrift/src/test/java/com/nicta/scoobi/io/thrift/MyThrift.java deleted file mode 100644 index 7960ed7c2..000000000 --- a/scoobi-thrift/src/test/java/com/nicta/scoobi/io/thrift/MyThrift.java +++ /dev/null @@ -1,388 +0,0 @@ -/** - * Autogenerated by Thrift Compiler (0.9.1) - * - * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING - * @generated - */ -package com.nicta.scoobi.io.thrift; - -import org.apache.thrift.scheme.IScheme; -import org.apache.thrift.scheme.SchemeFactory; -import org.apache.thrift.scheme.StandardScheme; - -import org.apache.thrift.scheme.TupleScheme; -import org.apache.thrift.protocol.TTupleProtocol; -import org.apache.thrift.protocol.TProtocolException; -import org.apache.thrift.EncodingUtils; -import org.apache.thrift.TException; -import org.apache.thrift.async.AsyncMethodCallback; -import org.apache.thrift.server.AbstractNonblockingServer.*; -import java.util.List; -import java.util.ArrayList; -import java.util.Map; -import java.util.HashMap; -import java.util.EnumMap; -import java.util.Set; -import java.util.HashSet; -import java.util.EnumSet; -import java.util.Collections; -import java.util.BitSet; -import java.nio.ByteBuffer; -import java.util.Arrays; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class MyThrift implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("MyThrift"); - - private static final org.apache.thrift.protocol.TField ENTITY_FIELD_DESC = new org.apache.thrift.protocol.TField("entity", org.apache.thrift.protocol.TType.STRING, (short)1); - - private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); - static { - schemes.put(StandardScheme.class, new MyThriftStandardSchemeFactory()); - schemes.put(TupleScheme.class, new MyThriftTupleSchemeFactory()); - } - - public String entity; // required - - /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ - public enum _Fields implements org.apache.thrift.TFieldIdEnum { - ENTITY((short)1, "entity"); - - private static final Map byName = new HashMap(); - - static { - for (_Fields field : EnumSet.allOf(_Fields.class)) { - byName.put(field.getFieldName(), field); - } - } - - /** - * Find the _Fields constant that matches fieldId, or null if its not found. - */ - public static _Fields findByThriftId(int fieldId) { - switch(fieldId) { - case 1: // ENTITY - return ENTITY; - default: - return null; - } - } - - /** - * Find the _Fields constant that matches fieldId, throwing an exception - * if it is not found. - */ - public static _Fields findByThriftIdOrThrow(int fieldId) { - _Fields fields = findByThriftId(fieldId); - if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); - return fields; - } - - /** - * Find the _Fields constant that matches name, or null if its not found. - */ - public static _Fields findByName(String name) { - return byName.get(name); - } - - private final short _thriftId; - private final String _fieldName; - - _Fields(short thriftId, String fieldName) { - _thriftId = thriftId; - _fieldName = fieldName; - } - - public short getThriftFieldId() { - return _thriftId; - } - - public String getFieldName() { - return _fieldName; - } - } - - // isset id assignments - public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; - static { - Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); - tmpMap.put(_Fields.ENTITY, new org.apache.thrift.meta_data.FieldMetaData("entity", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); - metaDataMap = Collections.unmodifiableMap(tmpMap); - org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(MyThrift.class, metaDataMap); - } - - public MyThrift() { - } - - public MyThrift( - String entity) - { - this(); - this.entity = entity; - } - - /** - * Performs a deep copy on other. - */ - public MyThrift(MyThrift other) { - if (other.isSetEntity()) { - this.entity = other.entity; - } - } - - public MyThrift deepCopy() { - return new MyThrift(this); - } - - @Override - public void clear() { - this.entity = null; - } - - public String getEntity() { - return this.entity; - } - - public MyThrift setEntity(String entity) { - this.entity = entity; - return this; - } - - public void unsetEntity() { - this.entity = null; - } - - /** Returns true if field entity is set (has been assigned a value) and false otherwise */ - public boolean isSetEntity() { - return this.entity != null; - } - - public void setEntityIsSet(boolean value) { - if (!value) { - this.entity = null; - } - } - - public void setFieldValue(_Fields field, Object value) { - switch (field) { - case ENTITY: - if (value == null) { - unsetEntity(); - } else { - setEntity((String)value); - } - break; - - } - } - - public Object getFieldValue(_Fields field) { - switch (field) { - case ENTITY: - return getEntity(); - - } - throw new IllegalStateException(); - } - - /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ - public boolean isSet(_Fields field) { - if (field == null) { - throw new IllegalArgumentException(); - } - - switch (field) { - case ENTITY: - return isSetEntity(); - } - throw new IllegalStateException(); - } - - @Override - public boolean equals(Object that) { - if (that == null) - return false; - if (that instanceof MyThrift) - return this.equals((MyThrift)that); - return false; - } - - public boolean equals(MyThrift that) { - if (that == null) - return false; - - boolean this_present_entity = true && this.isSetEntity(); - boolean that_present_entity = true && that.isSetEntity(); - if (this_present_entity || that_present_entity) { - if (!(this_present_entity && that_present_entity)) - return false; - if (!this.entity.equals(that.entity)) - return false; - } - - return true; - } - - @Override - public int hashCode() { - return 0; - } - - @Override - public int compareTo(MyThrift other) { - if (!getClass().equals(other.getClass())) { - return getClass().getName().compareTo(other.getClass().getName()); - } - - int lastComparison = 0; - - lastComparison = Boolean.valueOf(isSetEntity()).compareTo(other.isSetEntity()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetEntity()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.entity, other.entity); - if (lastComparison != 0) { - return lastComparison; - } - } - return 0; - } - - public _Fields fieldForId(int fieldId) { - return _Fields.findByThriftId(fieldId); - } - - public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { - schemes.get(iprot.getScheme()).getScheme().read(iprot, this); - } - - public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { - schemes.get(oprot.getScheme()).getScheme().write(oprot, this); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder("MyThrift("); - boolean first = true; - - sb.append("entity:"); - if (this.entity == null) { - sb.append("null"); - } else { - sb.append(this.entity); - } - first = false; - sb.append(")"); - return sb.toString(); - } - - public void validate() throws org.apache.thrift.TException { - // check for required fields - // check for sub-struct validity - } - - private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { - try { - write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { - try { - read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private static class MyThriftStandardSchemeFactory implements SchemeFactory { - public MyThriftStandardScheme getScheme() { - return new MyThriftStandardScheme(); - } - } - - private static class MyThriftStandardScheme extends StandardScheme { - - public void read(org.apache.thrift.protocol.TProtocol iprot, MyThrift struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TField schemeField; - iprot.readStructBegin(); - while (true) - { - schemeField = iprot.readFieldBegin(); - if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { - break; - } - switch (schemeField.id) { - case 1: // ENTITY - if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.entity = iprot.readString(); - struct.setEntityIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; - default: - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - iprot.readFieldEnd(); - } - iprot.readStructEnd(); - - // check for required fields of primitive type, which can't be checked in the validate method - struct.validate(); - } - - public void write(org.apache.thrift.protocol.TProtocol oprot, MyThrift struct) throws org.apache.thrift.TException { - struct.validate(); - - oprot.writeStructBegin(STRUCT_DESC); - if (struct.entity != null) { - oprot.writeFieldBegin(ENTITY_FIELD_DESC); - oprot.writeString(struct.entity); - oprot.writeFieldEnd(); - } - oprot.writeFieldStop(); - oprot.writeStructEnd(); - } - - } - - private static class MyThriftTupleSchemeFactory implements SchemeFactory { - public MyThriftTupleScheme getScheme() { - return new MyThriftTupleScheme(); - } - } - - private static class MyThriftTupleScheme extends TupleScheme { - - @Override - public void write(org.apache.thrift.protocol.TProtocol prot, MyThrift struct) throws org.apache.thrift.TException { - TTupleProtocol oprot = (TTupleProtocol) prot; - BitSet optionals = new BitSet(); - if (struct.isSetEntity()) { - optionals.set(0); - } - oprot.writeBitSet(optionals, 1); - if (struct.isSetEntity()) { - oprot.writeString(struct.entity); - } - } - - @Override - public void read(org.apache.thrift.protocol.TProtocol prot, MyThrift struct) throws org.apache.thrift.TException { - TTupleProtocol iprot = (TTupleProtocol) prot; - BitSet incoming = iprot.readBitSet(1); - if (incoming.get(0)) { - struct.entity = iprot.readString(); - struct.setEntityIsSet(true); - } - } - } - -} - diff --git a/scoobi-thrift/src/test/scala/com/nicta/scoobi/io/thrift/ThriftSchemaSpec.scala b/scoobi-thrift/src/test/scala/com/nicta/scoobi/io/thrift/ThriftSchemaSpec.scala deleted file mode 100644 index 945fada37..000000000 --- a/scoobi-thrift/src/test/scala/com/nicta/scoobi/io/thrift/ThriftSchemaSpec.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.nicta.scoobi.io.thrift - -import java.io._ - -import com.nicta.scoobi.Scoobi._ -import org.specs2.ScalaCheck -import org.specs2.mutable.Specification - -class ThriftSchemaSpec extends Specification with ScalaCheck { - - "WireFormat bidirectional" >> prop((s: String) => { - implicit val wf = implicitly[WireFormat[MyThrift]] - val a = new MyThrift(s) - val out = new ByteArrayOutputStream() - wf.toWire(a, new DataOutputStream(out)) - wf.fromWire(new DataInputStream(new ByteArrayInputStream(out.toByteArray))) ==== a - }) - - "SeqSchema bidirectional" >> prop((s: String) => { - implicit val ss = implicitly[SeqSchema[MyThrift]] - val a = new MyThrift(s) - ss.fromWritable(ss.toWritable(a)) ==== a - }) -} diff --git a/scoobi-thrift/src/test/thrift/build b/scoobi-thrift/src/test/thrift/build deleted file mode 100755 index 16c98d648..000000000 --- a/scoobi-thrift/src/test/thrift/build +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh -eu - -# This requires a copy of thrift to be installed because it can be run -# The expected files have been checked in for convenience - -DIR=$(dirname $0)/../../.. -thrift -r -out ${DIR}/src/test/java/ --gen java ${DIR}/src/test/thrift/test.thrift diff --git a/scoobi-thrift/src/test/thrift/test.thrift b/scoobi-thrift/src/test/thrift/test.thrift deleted file mode 100644 index 2b79d9f2f..000000000 --- a/scoobi-thrift/src/test/thrift/test.thrift +++ /dev/null @@ -1,5 +0,0 @@ -namespace java com.nicta.scoobi.io.thrift - -struct MyThrift { - 1: string entity; -}