import org.apache.spark.{ SparkConf, SparkContext }
import org.apache.spark.rdd.RDD
class BaseType(val a: String) extends Serializable {
override def toString = "(" + a + ")"
}
class TypeA(a: String, val b: String) extends BaseType(a) {
override def toString = "(" + a + "," + b + ")"
}
class TypeB(a: String, val b: String) extends BaseType(a) {
override def toString = "(" + a + "," + b + ")"
}
object EntityInheritance {
def main(args: Array[String]) = {
val sparkConf = new SparkConf()
.setMaster("local[*]")
.setAppName("EntityInheritance Sample")
val sc = new SparkContext(sparkConf)
val text_file = sc.textFile("/dqa/sample_logs/tipologies/entityInheritance.txt")
val items = text_file.flatMap(_.split("\n"))
val itemsRDDa = items.map(newInstanceA(_))
itemsRDDa.foreach { rdd => println(rdd) }
val countAa = countersAttributeA[TypeA](itemsRDDa)
val itemsRDDb = items.map(newInstanceB(_))
itemsRDDb.foreach { rdd => println(rdd) }
val countBa = countersAttributeA[TypeB](itemsRDDb)
sc.stop()
}
def newInstanceA(str: String): TypeA = {
val parts = str.split(" ")
new TypeA(parts(0), parts(1))
}
def newInstanceB(str: String): TypeB = {
val parts = str.split(" ")
new TypeB(parts(0), parts(1))
}
// I want to implement a generic function that receives RDD[TypeA] or RDD[TypeB]
// it's a simple example
def countersAttributeA[A](rdd: org.apache.spark.rdd.RDD[A]) = {
rdd
.map(s => (s.a, 1))
.reduceByKey(_ + _)
}
}
Hello, I have a problem but is possible that this idea is isn't good.
I trying to implement a generic function that receives different types. When create a different objects for example TypeA and TypeB I want to send to counterAttributeA -> count number of apparences of attribute 'a'
, but the console send this error:
[error] /src/main/scala/org/sparklambda/testing/EntityInheritance.scala:53: value a is not a member of type parameter A
[error] .map(s => (s.a, 1))
[error] ^
[error] one error found
Anyone you can help me? Thank's for all.
Copyright Notice:Content Author:「Diego Sanz」,Reproduced under the CC 4.0 BY-SA copyright license with a link to the original source and this disclaimer.
Link to original article:https://stackoverflow.com/questions/44436433/how-to-implement-generic-function-that-receives-differents-types-scala