Spark Structured Streaming is a powerful tool for reading and writing into the Kafka Topic To Read From the Kafka Topic that is secured by the SASL_SSL mechanism inputdf = (
spark
.readStream
.format("kafka")
.option("kafka.bootstrap.servers", BOOTSTRAP_SERVERS)
.option("kafka.sasl.jaas.config","org.apache.kafka.common.security.plain.PlainLoginModule required username='{}' password='{}';".format(USERNAME, PASSWORD))
.option("kafka.ssl.endpoint.identification.algorithm", "https")
.option("kafka.group.id", GROUPID)
.option("kafka.security.protocol", "SASL_SSL")
.option("kafka.sasl.mechanism", "PLAIN")
.option("subscribe", INPUT_TOPIC)
.option("latestFirst", True)
.option("startingOffsets", "earliest")
)
