[FFmpeg-cvslog] tls_openssl: Readd support for nonblocking operation

[Martin Storsjö]

ffmpeg | branch: master | Martin Storsjö <martin at martin.st> | Mon Jun 19 15:45:08 2017 +0300| [0671eb2346c17e8fb13784cf90ce416661fdea1c] | committer: Martin Storsjö

tls_openssl: Readd support for nonblocking operation

The rtmp protocol uses nonblocking reads, to poll for incoming
messages from the server while publishing a stream.

Prior to 94599a6de3822b13c94096d764868128f388ba28 and
d13b124eaf452b267480074b2e6946538ed03a6e, the tls protocol
handled the nonblocking flag, mostly as a side effect from not
using custom IO callbacks for reading from the socket. When custom
IO callbacks were taken into use in
d15eec4d6bdfa3bd4c4b5b7dd2dbd699ba253d02, the handling of a nonblocking
socket wasn't necessary for the default blocking mode any longer.

The code was simplified, since it was overlooked that other code
within libavformat actually used the tls protocol in nonblocking mode.

This fixes publishing over rtmps, with the openssl backend.

Signed-off-by: Martin Storsjö <martin at martin.st>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=0671eb2346c17e8fb13784cf90ce416661fdea1c
---

 libavformat/tls_openssl.c | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/libavformat/tls_openssl.c b/libavformat/tls_openssl.c
index 71fa96871d..f0b325ae98 100644
--- a/libavformat/tls_openssl.c
+++ b/libavformat/tls_openssl.c
@@ -109,6 +109,12 @@ void ff_openssl_deinit(void)
 
 static int print_tls_error(URLContext *h, int ret)
 {
+    TLSContext *c = h->priv_data;
+    if (h->flags & AVIO_FLAG_NONBLOCK) {
+        int err = SSL_get_error(c->ssl, ret);
+        if (err == SSL_ERROR_WANT_READ || err == SSL_ERROR_WANT_READ)
+            return AVERROR(EAGAIN);
+    }
     av_log(h, AV_LOG_ERROR, "%s\n", ERR_error_string(ERR_get_error(), NULL));
     return AVERROR(EIO);
 }
@@ -164,6 +170,8 @@ static int url_bio_bread(BIO *b, char *buf, int len)
     if (ret >= 0)
         return ret;
     BIO_clear_retry_flags(b);
+    if (ret == AVERROR(EAGAIN))
+        BIO_set_retry_read(b);
     if (ret == AVERROR_EXIT)
         return 0;
     return -1;
@@ -176,6 +184,8 @@ static int url_bio_bwrite(BIO *b, const char *buf, int len)
     if (ret >= 0)
         return ret;
     BIO_clear_retry_flags(b);
+    if (ret == AVERROR(EAGAIN))
+        BIO_set_retry_write(b);
     if (ret == AVERROR_EXIT)
         return 0;
     return -1;
@@ -292,7 +302,11 @@ fail:
 static int tls_read(URLContext *h, uint8_t *buf, int size)
 {
     TLSContext *c = h->priv_data;
-    int ret = SSL_read(c->ssl, buf, size);
+    int ret;
+    // Set or clear the AVIO_FLAG_NONBLOCK on c->tls_shared.tcp
+    c->tls_shared.tcp->flags &= ~AVIO_FLAG_NONBLOCK;
+    c->tls_shared.tcp->flags |= h->flags & AVIO_FLAG_NONBLOCK;
+    ret = SSL_read(c->ssl, buf, size);
     if (ret > 0)
         return ret;
     if (ret == 0)
@@ -303,7 +317,11 @@ static int tls_read(URLContext *h, uint8_t *buf, int size)
 static int tls_write(URLContext *h, const uint8_t *buf, int size)
 {
     TLSContext *c = h->priv_data;
-    int ret = SSL_write(c->ssl, buf, size);
+    int ret;
+    // Set or clear the AVIO_FLAG_NONBLOCK on c->tls_shared.tcp
+    c->tls_shared.tcp->flags &= ~AVIO_FLAG_NONBLOCK;
+    c->tls_shared.tcp->flags |= h->flags & AVIO_FLAG_NONBLOCK;
+    ret = SSL_write(c->ssl, buf, size);
     if (ret > 0)
         return ret;
     if (ret == 0)