[FFmpeg-devel] [PATCH v2 3/4] avcodec/ttmlenc: add initial support for regions and styles
Jan Ekström
jeebjp at gmail.com
Wed Apr 14 09:57:24 EEST 2021
From: Jan Ekström <jan.ekstrom at 24i.com>
Attempts to utilize the TTML cell resolution as a mapping to the
reference resolution, and maps font size to cell size. Additionally
sets the display and text alignment according to the ASS alignment
number.
Signed-off-by: Jan Ekström <jan.ekstrom at 24i.com>
---
libavcodec/ttmlenc.c | 210 ++++++++++++++++++++++++++++++++++---
libavcodec/ttmlenc.h | 3 +-
tests/ref/fate/sub-ttmlenc | 86 ++++++++-------
3 files changed, 247 insertions(+), 52 deletions(-)
diff --git a/libavcodec/ttmlenc.c b/libavcodec/ttmlenc.c
index e3c155fdd1..c190356683 100644
--- a/libavcodec/ttmlenc.c
+++ b/libavcodec/ttmlenc.c
@@ -100,20 +100,33 @@ static int ttml_encode_frame(AVCodecContext *avctx, uint8_t *buf,
dialog = ff_ass_split_dialog(s->ass_ctx, ass, 0, &num);
for (; dialog && num--; dialog++) {
- int ret = ff_ass_split_override_codes(&ttml_callbacks, s,
- dialog->text);
- int log_level = (ret != AVERROR_INVALIDDATA ||
- avctx->err_recognition & AV_EF_EXPLODE) ?
- AV_LOG_ERROR : AV_LOG_WARNING;
+ if (dialog->style) {
+ av_bprintf(&s->buffer, "<span region=\"");
+ av_bprint_escape(&s->buffer, dialog->style, NULL,
+ AV_ESCAPE_MODE_XML,
+ AV_ESCAPE_FLAG_XML_DOUBLE_QUOTES);
+ av_bprintf(&s->buffer, "\">");
+ }
- if (ret < 0) {
- av_log(avctx, log_level,
- "Splitting received ASS dialog failed: %s\n",
- av_err2str(ret));
+ {
+ int ret = ff_ass_split_override_codes(&ttml_callbacks, s,
+ dialog->text);
+ int log_level = (ret != AVERROR_INVALIDDATA ||
+ avctx->err_recognition & AV_EF_EXPLODE) ?
+ AV_LOG_ERROR : AV_LOG_WARNING;
- if (log_level == AV_LOG_ERROR)
- return ret;
+ if (ret < 0) {
+ av_log(avctx, log_level,
+ "Splitting received ASS dialog failed: %s\n",
+ av_err2str(ret));
+
+ if (log_level == AV_LOG_ERROR)
+ return ret;
+ }
}
+
+ if (dialog->style)
+ av_bprintf(&s->buffer, "</span>");
}
} else {
#endif
@@ -121,6 +134,14 @@ static int ttml_encode_frame(AVCodecContext *avctx, uint8_t *buf,
if (!dialog)
return AVERROR(ENOMEM);
+ if (dialog->style) {
+ av_bprintf(&s->buffer, "<span region=\"");
+ av_bprint_escape(&s->buffer, dialog->style, NULL,
+ AV_ESCAPE_MODE_XML,
+ AV_ESCAPE_FLAG_XML_DOUBLE_QUOTES);
+ av_bprintf(&s->buffer, "\">");
+ }
+
{
int ret = ff_ass_split_override_codes(&ttml_callbacks, s,
dialog->text);
@@ -140,6 +161,9 @@ static int ttml_encode_frame(AVCodecContext *avctx, uint8_t *buf,
}
}
+ if (dialog->style)
+ av_bprintf(&s->buffer, "</span>");
+
ff_ass_free_dialog(&dialog);
}
#if FF_API_ASS_TIMING
@@ -173,17 +197,175 @@ static av_cold int ttml_encode_close(AVCodecContext *avctx)
return 0;
}
+static const char *ttml_get_display_alignment(int alignment)
+{
+ switch (alignment) {
+ case 1:
+ case 2:
+ case 3:
+ return "after";
+ case 4:
+ case 5:
+ case 6:
+ return "center";
+ case 7:
+ case 8:
+ case 9:
+ return "before";
+ default:
+ return NULL;
+ }
+}
+
+static const char *ttml_get_text_alignment(int alignment)
+{
+ switch (alignment) {
+ case 1:
+ case 4:
+ case 7:
+ return "left";
+ case 2:
+ case 5:
+ case 8:
+ return "center";
+ case 3:
+ case 6:
+ case 9:
+ return "right";
+ default:
+ return NULL;
+ }
+}
+
+static int ttml_write_region(AVCodecContext *avctx, AVBPrint *buf,
+ ASSStyle *style)
+{
+ const char *display_alignment = NULL;
+ const char *text_alignment = NULL;
+
+ if (!style)
+ return AVERROR_INVALIDDATA;
+
+ if (!style->name) {
+ av_log(avctx, AV_LOG_ERROR, "Subtitle style name not set!\n");
+ return AVERROR_INVALIDDATA;
+ }
+
+ if (style->font_size < 0) {
+ av_log(avctx, AV_LOG_ERROR, "Invalid font size for TTML: %d!\n",
+ style->font_size);
+ return AVERROR_INVALIDDATA;
+ }
+
+ display_alignment = ttml_get_display_alignment(style->alignment);
+ text_alignment = ttml_get_text_alignment(style->alignment);
+ if (!display_alignment || !text_alignment) {
+ av_log(avctx, AV_LOG_ERROR,
+ "Failed to convert ASS style alignment %d of style %s to "
+ "TTML display and text alignment!\n",
+ style->alignment,
+ style->name);
+ return AVERROR_INVALIDDATA;
+ }
+
+ av_bprintf(buf, " <region xml:id=\"");
+ av_bprint_escape(buf, style->name, NULL, AV_ESCAPE_MODE_XML,
+ AV_ESCAPE_FLAG_XML_DOUBLE_QUOTES);
+ av_bprintf(buf, "\"\n");
+
+ av_bprintf(buf, " tts:displayAlign=\"");
+ av_bprint_escape(buf, display_alignment, NULL, AV_ESCAPE_MODE_XML,
+ AV_ESCAPE_FLAG_XML_DOUBLE_QUOTES);
+ av_bprintf(buf, "\"\n");
+
+ av_bprintf(buf, " tts:textAlign=\"");
+ av_bprint_escape(buf, text_alignment, NULL, AV_ESCAPE_MODE_XML,
+ AV_ESCAPE_FLAG_XML_DOUBLE_QUOTES);
+ av_bprintf(buf, "\"\n");
+
+ // if we set cell resolution to our script reference resolution,
+ // then a single line is a single "point" on our canvas. Thus, by setting
+ // our font size to font size in cells, we should gain a similar enough
+ // scale without resorting to explicit pixel based font sizing, which is
+ // frowned upon in the TTML community.
+ av_bprintf(buf, " tts:fontSize=\"%dc\"\n",
+ style->font_size);
+
+ if (style->font_name) {
+ av_bprintf(buf, " tts:fontFamily=\"");
+ av_bprint_escape(buf, style->font_name, NULL, AV_ESCAPE_MODE_XML,
+ AV_ESCAPE_FLAG_XML_DOUBLE_QUOTES);
+ av_bprintf(buf, "\"\n");
+ }
+
+ av_bprintf(buf, " tts:overflow=\"visible\" />\n");
+
+ return 0;
+}
+
static int ttml_write_header_content(AVCodecContext *avctx)
{
- if (!(avctx->extradata = av_mallocz(TTMLENC_EXTRADATA_SIGNATURE_SIZE +
- 1 + AV_INPUT_BUFFER_PADDING_SIZE))) {
+ TTMLContext *s = avctx->priv_data;
+ ASS *ass = (ASS *)s->ass_ctx;
+ ASSScriptInfo script_info = ass->script_info;
+ const size_t base_extradata_size = TTMLENC_EXTRADATA_SIGNATURE_SIZE + 1 +
+ AV_INPUT_BUFFER_PADDING_SIZE;
+ size_t additional_extradata_size = 0;
+
+ if (script_info.play_res_x <= 0 || script_info.play_res_y <= 0) {
+ av_log(avctx, AV_LOG_ERROR,
+ "Invalid subtitle reference resolution %dx%d!\n",
+ script_info.play_res_x, script_info.play_res_y);
+ return AVERROR_INVALIDDATA;
+ }
+
+ // write the first string in extradata, attributes in the base "tt" element.
+ av_bprintf(&s->buffer, ttml_default_namespacing);
+ // the cell resolution is in character cells, so not exactly 1:1 against
+ // a pixel based resolution, but as the tts:extent in the root
+ // "tt" element is frowned upon (and disallowed in the EBU-TT profile),
+ // we mimic the reference resolution by setting it as the cell resolution.
+ av_bprintf(&s->buffer, " ttp:cellResolution=\"%d %d\"\n",
+ script_info.play_res_x, script_info.play_res_y);
+ av_bprint_chars(&s->buffer, '\0', 1);
+
+ // write the second string in extradata, head element containing the styles
+ av_bprintf(&s->buffer, " <head>\n");
+ av_bprintf(&s->buffer, " <layout>\n");
+
+ for (int i = 0; i < ass->styles_count; i++) {
+ ASSStyle *style = &ass->styles[i];
+ int ret = ttml_write_region(avctx, &s->buffer, style);
+ if (ret < 0)
+ return ret;
+ }
+
+ av_bprintf(&s->buffer, " </layout>\n");
+ av_bprintf(&s->buffer, " </head>\n");
+ av_bprint_chars(&s->buffer, '\0', 1);
+
+ if (!av_bprint_is_complete(&s->buffer)) {
return AVERROR(ENOMEM);
}
- avctx->extradata_size = TTMLENC_EXTRADATA_SIGNATURE_SIZE;
+ additional_extradata_size = s->buffer.len;
+
+ if (!(avctx->extradata =
+ av_mallocz(base_extradata_size + additional_extradata_size))) {
+ return AVERROR(ENOMEM);
+ }
+
+ avctx->extradata_size =
+ TTMLENC_EXTRADATA_SIGNATURE_SIZE + additional_extradata_size;
memcpy(avctx->extradata, TTMLENC_EXTRADATA_SIGNATURE,
TTMLENC_EXTRADATA_SIGNATURE_SIZE);
+ if (additional_extradata_size)
+ memcpy(avctx->extradata + TTMLENC_EXTRADATA_SIGNATURE_SIZE,
+ s->buffer.str, additional_extradata_size);
+
+ av_bprint_clear(&s->buffer);
+
return 0;
}
diff --git a/libavcodec/ttmlenc.h b/libavcodec/ttmlenc.h
index c3bb11478d..467f35c7a6 100644
--- a/libavcodec/ttmlenc.h
+++ b/libavcodec/ttmlenc.h
@@ -28,6 +28,7 @@
static const char ttml_default_namespacing[] =
" xmlns=\"http://www.w3.org/ns/ttml\"\n"
" xmlns:ttm=\"http://www.w3.org/ns/ttml#metadata\"\n"
-" xmlns:tts=\"http://www.w3.org/ns/ttml#styling\"\n";
+" xmlns:tts=\"http://www.w3.org/ns/ttml#styling\"\n"
+" xmlns:ttp=\"http://www.w3.org/ns/ttml#parameter\"\n";
#endif /* AVCODEC_TTMLENC_H */
diff --git a/tests/ref/fate/sub-ttmlenc b/tests/ref/fate/sub-ttmlenc
index 51eab97817..6d0a8067fc 100644
--- a/tests/ref/fate/sub-ttmlenc
+++ b/tests/ref/fate/sub-ttmlenc
@@ -3,120 +3,132 @@
xmlns="http://www.w3.org/ns/ttml"
xmlns:ttm="http://www.w3.org/ns/ttml#metadata"
xmlns:tts="http://www.w3.org/ns/ttml#styling"
+ xmlns:ttp="http://www.w3.org/ns/ttml#parameter"
+ ttp:cellResolution="384 288"
xml:lang="">
+ <head>
+ <layout>
+ <region xml:id="Default"
+ tts:displayAlign="after"
+ tts:textAlign="center"
+ tts:fontSize="16c"
+ tts:fontFamily="Arial"
+ tts:overflow="visible" />
+ </layout>
+ </head>
<body>
<div>
<p
begin="00:00:00.000"
- end="00:00:00.000">Don't show this text it may be used to insert hidden data</p>
+ end="00:00:00.000"><span region="Default">Don't show this text it may be used to insert hidden data</span></p>
<p
begin="00:00:01.500"
- end="00:00:04.500">SubRip subtitles capability tester 1.3o by ale5000<br/>Use VLC 1.1 or higher as reference for most things and MPC Home Cinema for others<br/>This text should be blue<br/>This text should be red<br/>This text should be black<br/>If you see this with the normal font, the player don't (fully) support font face</p>
+ end="00:00:04.500"><span region="Default">SubRip subtitles capability tester 1.3o by ale5000<br/>Use VLC 1.1 or higher as reference for most things and MPC Home Cinema for others<br/>This text should be blue<br/>This text should be red<br/>This text should be black<br/>If you see this with the normal font, the player don't (fully) support font face</span></p>
<p
begin="00:00:04.500"
- end="00:00:04.500">Hidden</p>
+ end="00:00:04.500"><span region="Default">Hidden</span></p>
<p
begin="00:00:04.501"
- end="00:00:07.500">This text should be small<br/>This text should be normal<br/>This text should be big</p>
+ end="00:00:07.500"><span region="Default">This text should be small<br/>This text should be normal<br/>This text should be big</span></p>
<p
begin="00:00:07.501"
- end="00:00:11.500">This should be an E with an accent: È<br/>日本語<br/>This text should be bold, italics and underline<br/>This text should be small and green<br/>This text should be small and red<br/>This text should be big and brown</p>
+ end="00:00:11.500"><span region="Default">This should be an E with an accent: È<br/>日本語<br/>This text should be bold, italics and underline<br/>This text should be small and green<br/>This text should be small and red<br/>This text should be big and brown</span></p>
<p
begin="00:00:11.501"
- end="00:00:14.500">This line should be bold<br/>This line should be italics<br/>This line should be underline<br/>This line should be strikethrough<br/>Both lines<br/>should be underline</p>
+ end="00:00:14.500"><span region="Default">This line should be bold<br/>This line should be italics<br/>This line should be underline<br/>This line should be strikethrough<br/>Both lines<br/>should be underline</span></p>
<p
begin="00:00:14.501"
- end="00:00:17.500">><br/>It would be a good thing to<br/>hide invalid html tags that are closed and show the text in them<br/>but show un-closed invalid html tags<br/>Show not opened tags<br/><</p>
+ end="00:00:17.500"><span region="Default">><br/>It would be a good thing to<br/>hide invalid html tags that are closed and show the text in them<br/>but show un-closed invalid html tags<br/>Show not opened tags<br/><</span></p>
<p
begin="00:00:17.501"
- end="00:00:20.500">and also<br/>hide invalid html tags with parameters that are closed and show the text in them<br/>but show un-closed invalid html tags<br/>This text should be showed underlined without problems also: 2<3,5>1,4<6<br/>This shouldn't be underlined</p>
+ end="00:00:20.500"><span region="Default">and also<br/>hide invalid html tags with parameters that are closed and show the text in them<br/>but show un-closed invalid html tags<br/>This text should be showed underlined without problems also: 2<3,5>1,4<6<br/>This shouldn't be underlined</span></p>
<p
begin="00:00:20.501"
- end="00:00:21.500">This text should be in the normal position...</p>
+ end="00:00:21.500"><span region="Default">This text should be in the normal position...</span></p>
<p
begin="00:00:21.501"
- end="00:00:22.500">This text should NOT be in the normal position</p>
+ end="00:00:22.500"><span region="Default">This text should NOT be in the normal position</span></p>
<p
begin="00:00:22.501"
- end="00:00:24.500">Implementation is the same of the ASS tag<br/>This text should be at the<br/>top and horizontally centered</p>
+ end="00:00:24.500"><span region="Default">Implementation is the same of the ASS tag<br/>This text should be at the<br/>top and horizontally centered</span></p>
<p
begin="00:00:22.501"
- end="00:00:24.500">This text should be at the<br/>middle and horizontally centered</p>
+ end="00:00:24.500"><span region="Default">This text should be at the<br/>middle and horizontally centered</span></p>
<p
begin="00:00:22.501"
- end="00:00:24.500">This text should be at the<br/>bottom and horizontally centered</p>
+ end="00:00:24.500"><span region="Default">This text should be at the<br/>bottom and horizontally centered</span></p>
<p
begin="00:00:24.501"
- end="00:00:26.500">This text should be at the<br/>top and horizontally at the left</p>
+ end="00:00:26.500"><span region="Default">This text should be at the<br/>top and horizontally at the left</span></p>
<p
begin="00:00:24.501"
- end="00:00:26.500">This text should be at the<br/>middle and horizontally at the left<br/>(The second position must be ignored)</p>
+ end="00:00:26.500"><span region="Default">This text should be at the<br/>middle and horizontally at the left<br/>(The second position must be ignored)</span></p>
<p
begin="00:00:24.501"
- end="00:00:26.500">This text should be at the<br/>bottom and horizontally at the left</p>
+ end="00:00:26.500"><span region="Default">This text should be at the<br/>bottom and horizontally at the left</span></p>
<p
begin="00:00:26.501"
- end="00:00:28.500">This text should be at the<br/>top and horizontally at the right</p>
+ end="00:00:28.500"><span region="Default">This text should be at the<br/>top and horizontally at the right</span></p>
<p
begin="00:00:26.501"
- end="00:00:28.500">This text should be at the<br/>middle and horizontally at the right</p>
+ end="00:00:28.500"><span region="Default">This text should be at the<br/>middle and horizontally at the right</span></p>
<p
begin="00:00:26.501"
- end="00:00:28.500">This text should be at the<br/>bottom and horizontally at the right</p>
+ end="00:00:28.500"><span region="Default">This text should be at the<br/>bottom and horizontally at the right</span></p>
<p
begin="00:00:28.501"
- end="00:00:31.500">This could be the most difficult thing to implement</p>
+ end="00:00:31.500"><span region="Default">This could be the most difficult thing to implement</span></p>
<p
begin="00:00:31.501"
- end="00:00:50.500">First text</p>
+ end="00:00:50.500"><span region="Default">First text</span></p>
<p
begin="00:00:33.500"
- end="00:00:35.500">Second, it shouldn't overlap first</p>
+ end="00:00:35.500"><span region="Default">Second, it shouldn't overlap first</span></p>
<p
begin="00:00:35.501"
- end="00:00:37.500">Third, it should replace second</p>
+ end="00:00:37.500"><span region="Default">Third, it should replace second</span></p>
<p
begin="00:00:36.501"
- end="00:00:50.500">Fourth, it shouldn't overlap first and third</p>
+ end="00:00:50.500"><span region="Default">Fourth, it shouldn't overlap first and third</span></p>
<p
begin="00:00:40.501"
- end="00:00:45.500">Fifth, it should replace third</p>
+ end="00:00:45.500"><span region="Default">Fifth, it should replace third</span></p>
<p
begin="00:00:45.501"
- end="00:00:50.500">Sixth, it shouldn't be<br/>showed overlapped</p>
+ end="00:00:50.500"><span region="Default">Sixth, it shouldn't be<br/>showed overlapped</span></p>
<p
begin="00:00:50.501"
- end="00:00:52.500">TEXT 1 (bottom)</p>
+ end="00:00:52.500"><span region="Default">TEXT 1 (bottom)</span></p>
<p
begin="00:00:50.501"
- end="00:00:52.500">text 2</p>
+ end="00:00:52.500"><span region="Default">text 2</span></p>
<p
begin="00:00:52.501"
- end="00:00:54.500">Hide these tags:<br/>also hide these tags:<br/>but show this: {normal text}</p>
+ end="00:00:54.500"><span region="Default">Hide these tags:<br/>also hide these tags:<br/>but show this: {normal text}</span></p>
<p
begin="00:00:54.501"
- end="00:01:00.500"><br/>\ N is a forced line break<br/>\ h is a hard space<br/>Normal spaces at the start and at the end of the line are trimmed while hard spaces are not trimmed.<br/>The\hline\hwill\hnever\hbreak\hautomatically\hright\hbefore\hor\hafter\ha\hhard\hspace.\h:-D</p>
+ end="00:01:00.500"><span region="Default"><br/>\ N is a forced line break<br/>\ h is a hard space<br/>Normal spaces at the start and at the end of the line are trimmed while hard spaces are not trimmed.<br/>The\hline\hwill\hnever\hbreak\hautomatically\hright\hbefore\hor\hafter\ha\hhard\hspace.\h:-D</span></p>
<p
begin="00:00:54.501"
- end="00:00:56.500"><br/>\h\h\h\h\hA (05 hard spaces followed by a letter)<br/>A (Normal spaces followed by a letter)<br/>A (No hard spaces followed by a letter)</p>
+ end="00:00:56.500"><span region="Default"><br/>\h\h\h\h\hA (05 hard spaces followed by a letter)<br/>A (Normal spaces followed by a letter)<br/>A (No hard spaces followed by a letter)</span></p>
<p
begin="00:00:56.501"
- end="00:00:58.500">\h\h\h\h\hA (05 hard spaces followed by a letter)<br/>A (Normal spaces followed by a letter)<br/>A (No hard spaces followed by a letter)<br/>Show this: \TEST and this: \-)</p>
+ end="00:00:58.500"><span region="Default">\h\h\h\h\hA (05 hard spaces followed by a letter)<br/>A (Normal spaces followed by a letter)<br/>A (No hard spaces followed by a letter)<br/>Show this: \TEST and this: \-)</span></p>
<p
begin="00:00:58.501"
- end="00:01:00.500"><br/>A letter followed by 05 hard spaces: A\h\h\h\h\h<br/>A letter followed by normal spaces: A<br/>A letter followed by no hard spaces: A<br/>05 hard spaces between letters: A\h\h\h\h\hA<br/>5 normal spaces between letters: A A<br/><br/>^--Forced line break</p>
+ end="00:01:00.500"><span region="Default"><br/>A letter followed by 05 hard spaces: A\h\h\h\h\h<br/>A letter followed by normal spaces: A<br/>A letter followed by no hard spaces: A<br/>05 hard spaces between letters: A\h\h\h\h\hA<br/>5 normal spaces between letters: A A<br/><br/>^--Forced line break</span></p>
<p
begin="00:01:00.501"
- end="00:01:02.500">Both line should be strikethrough,<br/>yes.<br/>Correctly closed tags<br/>should be hidden.</p>
+ end="00:01:02.500"><span region="Default">Both line should be strikethrough,<br/>yes.<br/>Correctly closed tags<br/>should be hidden.</span></p>
<p
begin="00:01:02.501"
- end="00:01:04.500">It shouldn't be strikethrough,<br/>not opened tag showed as text.<br/>Not opened tag showed as text.</p>
+ end="00:01:04.500"><span region="Default">It shouldn't be strikethrough,<br/>not opened tag showed as text.<br/>Not opened tag showed as text.</span></p>
<p
begin="00:01:04.501"
- end="00:01:06.500">Three lines should be strikethrough,<br/>yes.<br/>Not closed tags showed as text</p>
+ end="00:01:06.500"><span region="Default">Three lines should be strikethrough,<br/>yes.<br/>Not closed tags showed as text</span></p>
<p
begin="00:01:06.501"
- end="00:01:08.500">Both line should be strikethrough but<br/>the wrong closing tag should be showed</p>
+ end="00:01:08.500"><span region="Default">Both line should be strikethrough but<br/>the wrong closing tag should be showed</span></p>
</div>
</body>
</tt>
--
2.30.2
More information about the ffmpeg-devel
mailing list