Scala Akka RestartSource onFailuresWithBackoff not working complaining dead letters encountered - akka-stream

Question

I have created a RetryFlow like below,following this article https://doc.akka.io/docs/akka/current/stream/operators/RetryFlow/withBackoff.html
private val httpFlow: Flow[HttpRequest, HttpResponse, Any] =
Http().outgoingConnection(host, port)
val retryFlow: Flow[HttpRequest, HttpResponse, Any] =
RetryFlow.withBackoff(minBackoff = 10.millis, maxBackoff = 5.seconds, randomFactor = 0d, maxRetries = 20, httpFlow)(
decideRetry = {
case (request: HttpRequest, response) =>
response.status match {
case StatusCodes.OK => Some(request)
case _ => None
}
})
And then processing flow like
val request = Post(Uri(path), some_json_payload)(
stringMarshaller(MediaTypes.`application/json`),
system.executionContext
)
httpClient.processRequest(request, retryFlow) map (...)
My http client looks like below, added a RestartSettings and restarting source on onFailuresWithBackoff
class HttpClient(implicit val system: ActorSystem[_]) {
implicit val materializer: Materializer = Materializer(system)
private def checkSuccess(response: HttpResponse, path: String): Either[Throwable, HttpResponse] =
Either.cond(
response.status == StatusCodes.OK,
response,
new Throwable(s"Could not GET $path, Service returned ${response.status.toString()}"),
)
private def httpRequest(
request: HttpRequest,
flow: Flow[HttpRequest, HttpResponse, Any],
): Future[Either[Throwable, HttpResponse]] =
Source.single(request).via(flow)
.runWith(Sink.head) map { response =>
checkSuccess(response, request.uri.path.toString())
}
private def unmarshallString(response: Either[Throwable, HttpResponse]): Future[Either[Throwable, String]] =
response.fold(
err => Future(err.asLeft[String]),
r => Unmarshal(r.entity).to[String].map(_.asRight[Throwable]),
)
def processRequest(request: HttpRequest, flow: Flow[HttpRequest, HttpResponse, Any]): Future[Enclosed[Json]] =
httpRequest(request, flow) flatMap unmarshallString map (_.flatMap(parse))
}
I am pretty new especially in akka , what I am missing here .