python - Un-persisting all dataframes in (py)spark

93

from pyspark.sql import SparkSession  spark = SparkSession.builder.getOrCreate ... spark.catalog.clearCache() 

from pyspark.sql import SQLContext from pyspark import SparkContext  sqlContext = SQLContext.getOrCreate(SparkContext.getOrCreate()) ... sqlContext.clearCache() 

85

for (id, rdd) in sc._jsc.getPersistentRDDs().items():     rdd.unpersist()     print("Unpersisted {} rdd".format(id))