[FFmpeg-devel] [PATCH v2] avformat: add a concat protocol that takes a line break delimited list of resources
Nicolas George
george at nsup.org
Sun Jun 27 21:26:19 EEST 2021
James Almer (12021-06-26):
> Suggested-by: ffmpeg at fb.com
> Signed-off-by: James Almer <jamrial at gmail.com>
> ---
> Updated documentation, and line breaks can now be part of the filename.
>
> doc/protocols.texi | 33 +++++++++
> libavformat/Makefile | 1 +
> libavformat/concat.c | 146 ++++++++++++++++++++++++++++++++++++++++
> libavformat/protocols.c | 1 +
> 4 files changed, 181 insertions(+)
>
> diff --git a/doc/protocols.texi b/doc/protocols.texi
> index ccdfb6e439..11de674225 100644
> --- a/doc/protocols.texi
> +++ b/doc/protocols.texi
> @@ -215,6 +215,39 @@ ffplay concat:split1.mpeg\|split2.mpeg\|split3.mpeg
> Note that you may need to escape the character "|" which is special for
> many shells.
>
> + at section concatf
> +
> +Physical concatenation protocol using a line break delimited list of
> +resources.
> +
> +Read and seek from many resources in sequence as if they were
> +a unique resource.
> +
> +A URL accepted by this protocol has the syntax:
> + at example
> +concatf:@var{URL}
> + at end example
> +
> +where @var{URL} is the url containing a line break delimited list of
> +resources to be concatenated, each one possibly specifying a distinct
> +protocol.
> +
> +For example to read a sequence of files @file{split1.mpeg},
> + at file{split2.mpeg}, @file{split3.mpeg} listed in separate lines within
> +a file @file{split.txt} with @command{ffplay} use the command:
> + at example
> +ffplay concatf:split.txt
> + at end example
> +Where @file{split.txt} contains the lines:
> + at example
> +split1.mpeg
> +split2.mpeg
> +split3.mpeg
> + at end example
> +
> +Note that if any of the entries in the list contain a line break as part
> +of their name, you'll need to escape it with a preceding "\" character.
This is not detailed and accurate enough. And it should be a link to the
common description of our escaping syntax anyway.
> +
> @section crypto
>
> AES-encrypted stream reading protocol.
> diff --git a/libavformat/Makefile b/libavformat/Makefile
> index c9ef564523..caca95802a 100644
> --- a/libavformat/Makefile
> +++ b/libavformat/Makefile
> @@ -616,6 +616,7 @@ OBJS-$(CONFIG_APPLEHTTP_PROTOCOL) += hlsproto.o
> OBJS-$(CONFIG_BLURAY_PROTOCOL) += bluray.o
> OBJS-$(CONFIG_CACHE_PROTOCOL) += cache.o
> OBJS-$(CONFIG_CONCAT_PROTOCOL) += concat.o
> +OBJS-$(CONFIG_CONCATF_PROTOCOL) += concat.o
> OBJS-$(CONFIG_CRYPTO_PROTOCOL) += crypto.o
> OBJS-$(CONFIG_DATA_PROTOCOL) += data_uri.o
> OBJS-$(CONFIG_FFRTMPCRYPT_PROTOCOL) += rtmpcrypt.o rtmpdigest.o rtmpdh.o
> diff --git a/libavformat/concat.c b/libavformat/concat.c
> index 278afd997d..b66e3b9e01 100644
> --- a/libavformat/concat.c
> +++ b/libavformat/concat.c
> @@ -22,9 +22,11 @@
> */
>
> #include "libavutil/avstring.h"
> +#include "libavutil/bprint.h"
> #include "libavutil/mem.h"
>
> #include "avformat.h"
> +#include "avio_internal.h"
> #include "url.h"
>
> #define AV_CAT_SEPARATOR "|"
> @@ -56,6 +58,7 @@ static av_cold int concat_close(URLContext *h)
> return err < 0 ? -1 : 0;
> }
>
> +#if CONFIG_CONCAT_PROTOCOL
> static av_cold int concat_open(URLContext *h, const char *uri, int flags)
> {
> char *node_uri = NULL;
> @@ -124,6 +127,7 @@ static av_cold int concat_open(URLContext *h, const char *uri, int flags)
> data->total_size = total_size;
> return err;
> }
> +#endif
>
> static int concat_read(URLContext *h, unsigned char *buf, int size)
> {
> @@ -188,6 +192,7 @@ static int64_t concat_seek(URLContext *h, int64_t pos, int whence)
> return result;
> }
>
> +#if CONFIG_CONCAT_PROTOCOL
> const URLProtocol ff_concat_protocol = {
> .name = "concat",
> .url_open = concat_open,
> @@ -197,3 +202,144 @@ const URLProtocol ff_concat_protocol = {
> .priv_data_size = sizeof(struct concat_data),
> .default_whitelist = "concat,file,subfile",
> };
> +#endif
> +
> +#if CONFIG_CONCATF_PROTOCOL
> +// Custom ff_read_line_to_bprint() implementation where line breaks can be
> +// part of the line being read if escaped.
> +static int64_t read_line_to_bprint(AVIOContext *s, AVBPrint *bp)
> +{
> + int len, end;
> + int64_t read = 0;
> + char tmp[1024];
> + char c;
> +
> + do {
> + len = 0;
> + do {
> + char escape = c = avio_r8(s);
> + if (c == '\\')
> + c = avio_r8(s);
> + end = (c == '\r' || c == '\n' || c == '\0');
> + if (end && escape == '\\') {
> + if (c != '\0') {
> + tmp[len++] = c;
> + end = 0;
> + } else
> + tmp[len++] = escape;
> + } else if (!end) {
> + if (escape == '\\') {
> + tmp[len++] = escape;
> + avio_skip(s, -1);
> + } else
> + tmp[len++] = c;
> + }
> + } while (!end && len < sizeof(tmp));
> + av_bprint_append_data(bp, tmp, len);
> + read += len;
> + } while (!end);
> +
> + if (c == '\r' && avio_r8(s) != '\n' && !avio_feof(s))
> + avio_skip(s, -1);
> +
> + if (!c && s->error)
> + return s->error;
> +
> + if (!c && !read && avio_feof(s))
> + return AVERROR_EOF;
> +
> + return read;
> +}
Re-implementing yet another de-escaping and splitting function is a
terrible idea, I am strongly against it.
It would be easier if we had a good string API already. Barring that, I
think you need to load the whole file and use the existing de-escaping
functions.
> +
> +static av_cold int concatf_open(URLContext *h, const char *uri, int flags)
> +{
> + AVBPrint bp;
> + struct concat_data *data = h->priv_data;
> + struct concat_nodes *nodes = NULL;
> + AVIOContext *in = NULL;
> + URLContext *uc;
> + int64_t size, total_size = 0;
> + unsigned int nodes_size = 0;
> + size_t i = 0;
> + int err = 0;
> +
> + if (!av_strstart(uri, "concatf:", &uri)) {
> + av_log(h, AV_LOG_ERROR, "URL %s lacks prefix\n", uri);
> + return AVERROR(EINVAL);
> + }
> +
> + /* handle input */
> + if (!*uri)
> + return AVERROR(ENOENT);
> +
> + err = ffio_open_whitelist(&in, uri, AVIO_FLAG_READ, &h->interrupt_callback,
> + NULL, h->protocol_whitelist, h->protocol_blacklist);
> + if (err < 0)
> + return err;
> +
> + av_bprint_init(&bp, 0, AV_BPRINT_SIZE_UNLIMITED);
> +
> + for (i = 0; !avio_feof(in); i++) {
> + size_t len = i;
> +
> + av_bprint_clear(&bp);
> + if ((err = read_line_to_bprint(in, &bp)) <= 0) {
> + if (err == 0 && i == 0)
> + err = AVERROR_INVALIDDATA;
> + else if (err == AVERROR_EOF)
> + err = 0;
> + break;
> + }
> +
> + if (++len == SIZE_MAX / sizeof(*nodes)) {
> + err = AVERROR(ENAMETOOLONG);
> + break;
> + }
> +
> + /* creating URLContext */
> + err = ffurl_open_whitelist(&uc, bp.str, flags,
> + &h->interrupt_callback, NULL, h->protocol_whitelist, h->protocol_blacklist, h);
> + if (err < 0)
> + break;
> +
> + /* creating size */
> + if ((size = ffurl_size(uc)) < 0) {
> + ffurl_close(uc);
> + err = AVERROR(ENOSYS);
> + break;
> + }
> +
> + nodes = av_fast_realloc(data->nodes, &nodes_size, sizeof(*nodes) * len);
> + if (!nodes) {
> + ffurl_close(uc);
> + err = AVERROR(ENOMEM);
> + break;
> + }
> + data->nodes = nodes;
> +
> + /* assembling */
> + data->nodes[i].uc = uc;
> + data->nodes[i].size = size;
> + total_size += size;
> + }
> + avio_closep(&in);
> + av_bprint_finalize(&bp, NULL);
> + data->length = i;
> +
> + if (err < 0)
> + concat_close(h);
> +
> + data->total_size = total_size;
> + return err;
> +}
> +
> +const URLProtocol ff_concatf_protocol = {
> + .name = "concatf",
> + .url_open = concatf_open,
> + .url_read = concat_read,
> + .url_seek = concat_seek,
> + .url_close = concat_close,
> + .priv_data_size = sizeof(struct concat_data),
> + .default_whitelist = "concatf,concat,file,subfile",
> +};
> +#endif
> diff --git a/libavformat/protocols.c b/libavformat/protocols.c
> index 4b6b1c8e98..7f08f151b6 100644
> --- a/libavformat/protocols.c
> +++ b/libavformat/protocols.c
> @@ -27,6 +27,7 @@ extern const URLProtocol ff_async_protocol;
> extern const URLProtocol ff_bluray_protocol;
> extern const URLProtocol ff_cache_protocol;
> extern const URLProtocol ff_concat_protocol;
> +extern const URLProtocol ff_concatf_protocol;
> extern const URLProtocol ff_crypto_protocol;
> extern const URLProtocol ff_data_protocol;
> extern const URLProtocol ff_ffrtmpcrypt_protocol;
Regards,
--
Nicolas George
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 833 bytes
Desc: not available
URL: <https://ffmpeg.org/pipermail/ffmpeg-devel/attachments/20210627/f28ea916/attachment.sig>
More information about the ffmpeg-devel
mailing list