[FFmpeg-devel] [PATCH v3 1/1] avformat: Add IPFS protocol support.
Mark Gaiser
markg85 at gmail.com
Thu Feb 3 19:26:20 EET 2022
Ignore this one, the title went wrong.
On Thu, Feb 3, 2022 at 6:26 PM Mark Gaiser <markg85 at gmail.com> wrote:
> This patch adds support for:
> - ffplay ipfs://<cid>
> - ffplay ipns://<cid>
>
> IPFS data can be played from so called "ipfs gateways".
> A gateway is essentially a webserver that gives access to the
> distributed IPFS network.
>
> This protocol support (ipfs and ipns) therefore translates
> ipfs:// and ipns:// to a http:// url. This resulting url is
> then handled by the http protocol. It could also be https
> depending on the gateway provided.
>
> To use this protocol, a gateway must be provided.
> If you do nothing it will try to find it in your
> $HOME/.ipfs/gateway file. The ways to set it manually are:
> 1. Define a -gateway <url> to the gateway.
> 2. Define $IPFS_GATEWAY with the full http link to the gateway.
> 3. Define $IPFS_PATH and point it to the IPFS data path.
> 4. Have IPFS running in your local user folder (under $HOME/.ipfs).
>
> Signed-off-by: Mark Gaiser <markg85 at gmail.com>
> ---
> configure | 2 +
> doc/protocols.texi | 30 ++++
> libavformat/Makefile | 2 +
> libavformat/ipfsgateway.c | 329 ++++++++++++++++++++++++++++++++++++++
> libavformat/protocols.c | 2 +
> 5 files changed, 365 insertions(+)
> create mode 100644 libavformat/ipfsgateway.c
>
> diff --git a/configure b/configure
> index 5b19a35f59..6ff09e7974 100755
> --- a/configure
> +++ b/configure
> @@ -3585,6 +3585,8 @@ udp_protocol_select="network"
> udplite_protocol_select="network"
> unix_protocol_deps="sys_un_h"
> unix_protocol_select="network"
> +ipfs_protocol_select="https_protocol"
> +ipns_protocol_select="https_protocol"
>
> # external library protocols
> libamqp_protocol_deps="librabbitmq"
> diff --git a/doc/protocols.texi b/doc/protocols.texi
> index d207df0b52..7c9c0a4808 100644
> --- a/doc/protocols.texi
> +++ b/doc/protocols.texi
> @@ -2025,5 +2025,35 @@ decoding errors.
>
> @end table
>
> + at section ipfs
> +
> +InterPlanetary File System (IPFS) protocol support. One can access files
> stored
> +on the IPFS network through so called gateways. Those are http(s)
> endpoints.
> +This protocol wraps the IPFS native protocols (ipfs:// and ipns://) to be
> send
> +to such a gateway. Users can (and should) host their own node which means
> this
> +protocol will use your local machine gateway to access files on the IPFS
> network.
> +
> +If a user doesn't have a node of their own then the public gateway
> dweb.link is
> +used by default.
> +
> +You can use this protocol in 2 ways. Using IPFS:
> + at example
> +ffplay ipfs://QmbGtJg23skhvFmu9mJiePVByhfzu5rwo74MEkVDYAmF5T
> + at end example
> +
> +Or the IPNS protocol (IPNS is mutable IPFS):
> + at example
> +ffplay ipns://QmbGtJg23skhvFmu9mJiePVByhfzu5rwo74MEkVDYAmF5T
> + at end example
> +
> +You can also change the gateway to be used:
> +
> + at table @option
> +
> + at item gateway
> +Defines the gateway to use. When nothing is provided the protocol will
> first try
> +your local gateway. If that fails dweb.link will be used.
> +
> + at end table
>
> @c man end PROTOCOLS
> diff --git a/libavformat/Makefile b/libavformat/Makefile
> index 3dc6a479cc..4edce8420f 100644
> --- a/libavformat/Makefile
> +++ b/libavformat/Makefile
> @@ -656,6 +656,8 @@ OBJS-$(CONFIG_SRTP_PROTOCOL) +=
> srtpproto.o srtp.o
> OBJS-$(CONFIG_SUBFILE_PROTOCOL) += subfile.o
> OBJS-$(CONFIG_TEE_PROTOCOL) += teeproto.o tee_common.o
> OBJS-$(CONFIG_TCP_PROTOCOL) += tcp.o
> +OBJS-$(CONFIG_IPFS_PROTOCOL) += ipfsgateway.o
> +OBJS-$(CONFIG_IPNS_PROTOCOL) += ipfsgateway.o
> TLS-OBJS-$(CONFIG_GNUTLS) += tls_gnutls.o
> TLS-OBJS-$(CONFIG_LIBTLS) += tls_libtls.o
> TLS-OBJS-$(CONFIG_MBEDTLS) += tls_mbedtls.o
> diff --git a/libavformat/ipfsgateway.c b/libavformat/ipfsgateway.c
> new file mode 100644
> index 0000000000..72d431eac3
> --- /dev/null
> +++ b/libavformat/ipfsgateway.c
> @@ -0,0 +1,329 @@
> +/*
> + * IPFS and IPNS protocol support through IPFS Gateway.
> + * Copyright (c) 2022 Mark Gaiser
> + *
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
> 02110-1301 USA
> + */
> +
> +#include "avformat.h"
> +#include "libavutil/avassert.h"
> +#include "libavutil/avstring.h"
> +#include "libavutil/internal.h"
> +#include "libavutil/opt.h"
> +#include "libavutil/tree.h"
> +#include <fcntl.h>
> +#if HAVE_IO_H
> +#include <io.h>
> +#endif
> +#if HAVE_UNISTD_H
> +#include <unistd.h>
> +#endif
> +#include "os_support.h"
> +#include "url.h"
> +#include <stdlib.h>
> +#include <sys/stat.h>
> +
> +typedef struct IPFSGatewayContext {
> + AVClass *class;
> + URLContext *inner;
> + char *gateway;
> +} IPFSGatewayContext;
> +
> +// A best-effort way to find the IPFS gateway.
> +// Only the most appropiate gateway is set. It's not actually requested
> +// (http call) to prevent a potential slowdown in startup. A potential
> timeout
> +// is handled by the HTTP protocol.
> +//
> +// Return codes can be:
> +// 1 : A potential gateway is found and set in c->gateway
> +// -1: The IPFS data folder could not be found
> +// -2: The gateway file could not be found
> +// -3: The gateway file is found but empty
> +// -4: $HOME is empty
> +// -9: Unhandled error
> +static int populate_ipfs_gateway(URLContext *h)
> +{
> + IPFSGatewayContext *c = h->priv_data;
> + char *ipfs_full_data_folder = NULL;
> + char *ipfs_gateway_file = NULL;
> + struct stat st;
> + int stat_ret = 0;
> + int ret = -9;
> + FILE *gateway_file = NULL;
> + char gateway_file_data[1000];
> +
> + // First, test if there already is a path in c->gateway. If it is
> then it
> + // was provided as cli arument and should be used. It takes precdence.
> + if (c->gateway != NULL) {
> + ret = 1;
> + goto err;
> + }
> +
> + // Test $IPFS_GATEWAY.
> + if (getenv("IPFS_GATEWAY") != NULL) {
> + av_free(c->gateway);
> + c->gateway = av_asprintf("%s", getenv("IPFS_GATEWAY"));
> + ret = 1;
> + goto err;
> + } else
> + av_log(h, AV_LOG_DEBUG, "$IPFS_GATEWAY is empty.\n");
> +
> + // We need to know the IPFS folder to - eventually - read the
> contents of
> + // the "gateway" file which would tell us the gateway to use.
> + if (getenv("IPFS_PATH") == NULL) {
> + av_log(h, AV_LOG_DEBUG, "$IPFS_PATH is empty.\n");
> +
> + // Try via the home folder.
> + if (getenv("HOME") == NULL) {
> + av_log(h, AV_LOG_ERROR, "$HOME appears to be empty.\n");
> + ret = -4;
> + goto err;
> + }
> +
> + ipfs_full_data_folder = av_asprintf("%s/.ipfs/", getenv("HOME"));
> +
> + // Stat the folder.
> + // It should exist in a default IPFS setup when run as local user.
> +#ifndef _WIN32
> + stat_ret = stat(ipfs_full_data_folder, &st);
> +#else
> + stat_ret = win32_stat(ipfs_full_data_folder, &st);
> +#endif
> + if (stat_ret < 0) {
> + av_log(h, AV_LOG_DEBUG, "Unable to find IPFS folder. We
> tried:\n");
> + av_log(h, AV_LOG_DEBUG, "- $IPFS_PATH, which was empty.\n");
> + av_log(h, AV_LOG_DEBUG, "- $HOME/.ipfs (full uri: %s) which
> doesn't exist.\n", ipfs_full_data_folder);
> + ret = -1;
> + goto err;
> + }
> + } else
> + ipfs_full_data_folder = av_asprintf("%s", getenv("IPFS_PATH"));
> +
> + ipfs_gateway_file = av_asprintf("%sgateway", ipfs_full_data_folder);
> +
> + // Get the contents of the gateway file.
> + gateway_file = av_fopen_utf8(ipfs_gateway_file, "r");
> + if (!gateway_file) {
> + av_log(h, AV_LOG_ERROR, "The IPFS gateway file (full uri: %s)
> doesn't exist. Is the gateway enabled?\n", ipfs_gateway_file);
> + ret = -2;
> + goto err;
> + }
> +
> + // Read a single line (fgets stops at new line mark).
> + fgets(gateway_file_data, sizeof(gateway_file_data) - 1, gateway_file);
> +
> + // Replace first occurence of end of line to \0
> + gateway_file_data[strcspn(gateway_file_data, "\r\n")] = 0;
> +
> + // If strlen finds anything longer then 0 characters then we have a
> + // potential gateway url.
> + if (strlen(gateway_file_data) < 1) {
> + av_log(h, AV_LOG_ERROR, "The IPFS gateway file (full uri: %s)
> appears to be empty. Is the gateway started?\n", ipfs_gateway_file);
> + ret = -3;
> + goto err;
> + }
> +
> + // At this point gateway_file_data contains at least something.
> + // Copy it into c->gateway.
> + c->gateway = av_asprintf("%s", gateway_file_data);
> + if (c->gateway) {
> + ret = 1;
> + goto err;
> + } else
> + av_log(h, AV_LOG_DEBUG, "Unknown error in the IPFS gateway
> file.\n");
> +
> +err:
> + if (gateway_file)
> + fclose(gateway_file);
> +
> + av_free(ipfs_full_data_folder);
> + av_free(ipfs_gateway_file);
> + return ret;
> +}
> +
> +// For now just makes sure that the gateway ends in url we expect.
> +// Like http://localhost:8080/.
> +// Explicitly with the traling slash.
> +//
> +// Return codes can be: (any code above 0 is OK)
> +// 2 : c->gateway modified to add a trailing "/"
> +// 1 : c->gateway is set with a string that looks to be a valid URL
> +// -1: The gateway is too short to be any kind of URL.
> +// -2: The gateway didn't start with http:// or https://
> +// -3: The gateway url part (without the protocol) is too short. We
> expect 3
> +// characters minimal. So http://aaa would be the bare minimal.
> +// -9: Unhandled error
> +static int sanitize_ipfs_gateway(URLContext *h)
> +{
> + IPFSGatewayContext *c = h->priv_data;
> + char last_gateway_char;
> + const char *url_without_protocol;
> + const char *uri = c->gateway;
> + int ret = -9;
> +
> + // "http://" is already 7 characters. Lets be really conservative
> and say a
> + // url must be 10 chatacters at the very least.
> + if (strlen(c->gateway) < 10) {
> + av_log(h, AV_LOG_ERROR, "The gateway URL must be at least 10
> characters long.\n");
> + ret = -1;
> + goto err;
> + }
> +
> + // Test if the gateway starts with either http:// or https://
> + // The remainder is stored in url_without_protocol
> + if (av_stristart(uri, "http://", &url_without_protocol) == 0
> + && av_stristart(uri, "https://", &url_without_protocol) == 0) {
> + av_log(h, AV_LOG_ERROR, "The gateway URL didn't start with http://
> or https:// and is therefore invalid.\n");
> + ret = -2;
> + goto err;
> + }
> +
> + // We now know the remainder of the url without the protocol. Check
> it for
> + // some length. At least 3 characters.
> + if (strlen(url_without_protocol) < 3) {
> + av_log(h, AV_LOG_ERROR, "The gateway url (without the protocol
> part) is too short to be a valid URL.\n");
> + ret = -3;
> + goto err;
> + }
> +
> + last_gateway_char = c->gateway[strlen(c->gateway) - 1];
> +
> + if (last_gateway_char != '/') {
> + c->gateway = av_asprintf("%s/", c->gateway);
> + ret = 2;
> + goto err;
> + }
> +
> +err:
> + return ret;
> +}
> +
> +static int translate_ipfs_to_http(URLContext *h, const char *uri,
> + int flags, AVDictionary **options)
> +{
> + const char *ipfs_cid;
> + const char *protocol_path_suffix = "ipfs/";
> + char *fulluri = NULL;
> + int ret;
> + IPFSGatewayContext *c = h->priv_data;
> +
> + // Test for ipfs://, ipfs:, ipns:// and ipns:. This prefix is
> stripped from
> + // the string leaving just the CID in ipfs_cid.
> + int is_ipfs = (av_stristart(uri, "ipfs://", &ipfs_cid)
> + || av_stristart(uri, "ipfs:", &ipfs_cid));
> + int is_ipns = (av_stristart(uri, "ipns://", &ipfs_cid)
> + || av_stristart(uri, "ipns:", &ipfs_cid));
> +
> + // Populate the IPFS gateway if we have any.
> + // If not, inform the user how to properly set one.
> + if (populate_ipfs_gateway(h) < 0) {
> + av_log(h, AV_LOG_ERROR, "No IPFS gateway was set. Make sure a
> local IPFS instance is running.\n");
> + av_log(h, AV_LOG_INFO, "There are multiple options to define this
> gateway. The below options are in order of precedence:\n");
> + av_log(h, AV_LOG_INFO, "1. Define a -gateway <url> to the gateway
> without trailing forward slash.\n");
> + av_log(h, AV_LOG_INFO, "2. Define $IPFS_GATEWAY with the full
> http link to the gateway without trailing forward slash.\n");
> + av_log(h, AV_LOG_INFO, "3. Define $IPFS_PATH and point it to the
> IPFS data path.\n");
> + av_log(h, AV_LOG_INFO, "4. Have IPFS running in your local user
> folder (under $HOME/.ipfs).\n");
> + av_log(h, AV_LOG_INFO, "In all path cases, a file named gateway
> is expected. See https://github.com/ipfs/specs/issues/261 for more
> information.\n");
> + ret = AVERROR(EINVAL);
> + goto err;
> + }
> +
> + // Sanitize the gateway to a format we expect.
> + if (sanitize_ipfs_gateway(h) < 1)
> + goto err;
> +
> + // We must have either ipns or ipfs.
> + if (!is_ipfs && !is_ipns) {
> + ret = AVERROR(EINVAL);
> + av_log(h, AV_LOG_ERROR, "Unsupported url %s\n", uri);
> + goto err;
> + }
> +
> + // If we have IPNS, update the protocol.
> + if (is_ipns)
> + protocol_path_suffix = "ipns/";
> +
> + // Concatenate the url.
> + // This ends up with something like: http://localhost:8080/ipfs/Qm...
> ..
> + fulluri = av_asprintf("%s%s%s", c->gateway, protocol_path_suffix,
> ipfs_cid);
> +
> + // Pass the URL back to FFMpeg's protocol handler.
> + if ((ret = ffurl_open_whitelist(&c->inner, fulluri, flags,
> + &h->interrupt_callback, options,
> + h->protocol_whitelist,
> + h->protocol_blacklist, h))
> + < 0) {
> + av_log(h, AV_LOG_ERROR, "Unable to open resource: %s\n", fulluri);
> + goto err;
> + }
> +
> +err:
> + av_free(fulluri);
> + return ret;
> +}
> +
> +static int ipfs_read(URLContext *h, unsigned char *buf, int size)
> +{
> + IPFSGatewayContext *c = h->priv_data;
> + return ffurl_read(c->inner, buf, size);
> +}
> +
> +static int64_t ipfs_seek(URLContext *h, int64_t pos, int whence)
> +{
> + IPFSGatewayContext *c = h->priv_data;
> + return ffurl_seek(c->inner, pos, whence);
> +}
> +
> +static int ipfs_close(URLContext *h)
> +{
> + IPFSGatewayContext *c = h->priv_data;
> + return ffurl_closep(&c->inner);
> +}
> +
> +#define OFFSET(x) offsetof(IPFSGatewayContext, x)
> +
> +static const AVOption options[] = {
> + {"gateway", "The gateway to ask for IPFS data.", OFFSET(gateway),
> AV_OPT_TYPE_STRING, {.str = NULL}, 0, 0, AV_OPT_FLAG_DECODING_PARAM},
> + {NULL},
> +};
> +
> +static const AVClass ipfs_context_class = {
> + .class_name = "IPFS",
> + .item_name = av_default_item_name,
> + .option = options,
> + .version = LIBAVUTIL_VERSION_INT,
> +};
> +
> +const URLProtocol ff_ipfs_protocol = {
> + .name = "ipfs",
> + .url_open2 = translate_ipfs_to_http,
> + .url_read = ipfs_read,
> + .url_seek = ipfs_seek,
> + .url_close = ipfs_close,
> + .priv_data_size = sizeof(IPFSGatewayContext),
> + .priv_data_class = &ipfs_context_class,
> +};
> +
> +const URLProtocol ff_ipns_protocol = {
> + .name = "ipns",
> + .url_open2 = translate_ipfs_to_http,
> + .url_read = ipfs_read,
> + .url_seek = ipfs_seek,
> + .url_close = ipfs_close,
> + .priv_data_size = sizeof(IPFSGatewayContext),
> + .priv_data_class = &ipfs_context_class,
> +};
> diff --git a/libavformat/protocols.c b/libavformat/protocols.c
> index 948fae411f..675b684bd3 100644
> --- a/libavformat/protocols.c
> +++ b/libavformat/protocols.c
> @@ -73,6 +73,8 @@ extern const URLProtocol ff_libsrt_protocol;
> extern const URLProtocol ff_libssh_protocol;
> extern const URLProtocol ff_libsmbclient_protocol;
> extern const URLProtocol ff_libzmq_protocol;
> +extern const URLProtocol ff_ipfs_protocol;
> +extern const URLProtocol ff_ipns_protocol;
>
> #include "libavformat/protocol_list.c"
>
> --
> 2.35.1
>
>
More information about the ffmpeg-devel
mailing list