spark-sql性能优化之——多线程实现多Job并发执行

直接上代码

val spark = SparkSession.builder() .appName("name") .master("local[2]") .getOrCreate() val df = spark.read.json("src\\main\\resources\\json.txt") df.show() //没有多线程处理的状况，连续执行两个Action操做，生成两个Job
  df.rdd.saveAsTextFile("") df.rdd.saveAsTextFile("") //用Executor实现多线程方式处理Job
  val dfList = Array(df,df) val executorService = Executors.newFixedThreadPool(2) for(df <- dfList) { executorService.submit(new Callable[Boolean]() { def call() : Boolean = { df.show() true } }) } executorService.shutdown() spark.stop()