[FFmpeg-devel,v3,3/3] avformat/hashenc: add streamhash muxer

Submitted by Moritz Barsnick on Sept. 12, 2019, 9:23 a.m.

Details

Message ID 8e539006873d44a2670b5d55bae06d79cc0eb999.1568279966.git.barsnick@gmx.net
State New
Headers show

Commit Message

Moritz Barsnick Sept. 12, 2019, 9:23 a.m.
Implemented as a variant of the hash muxer, reusing most functions,
and making use of the previously introduced array of hashes.

Signed-off-by: Moritz Barsnick <barsnick@gmx.net>
---
 Changelog                |   1 +
 doc/muxers.texi          |  47 ++++++++++++++++++
 libavformat/Makefile     |   1 +
 libavformat/allformats.c |   1 +
 libavformat/hashenc.c    | 100 +++++++++++++++++++++++++++++++++++----
 libavformat/version.h    |   4 +-
 6 files changed, 142 insertions(+), 12 deletions(-)

--
2.20.1

Patch hide | download patch | download mbox

diff --git a/Changelog b/Changelog
index 4b29e015a0..8eeed65680 100644
--- a/Changelog
+++ b/Changelog
@@ -9,6 +9,7 @@  version <next>:
 - Supoort AMD AMF encoder on Linux (via Vulkan)
 - IMM5 video decoder
 - ZeroMQ protocol
+- streamhash muxer


 version 4.2:
diff --git a/doc/muxers.texi b/doc/muxers.texi
index 20d31b279c..eb989f888b 100644
--- a/doc/muxers.texi
+++ b/doc/muxers.texi
@@ -2064,6 +2064,53 @@  Specify whether to remove all fragments when finished. Default 0 (do not remove)

 @end table

+@anchor{streamhash}
+@section streamhash
+
+Per stream hash testing format.
+
+This muxer computes and prints a cryptographic hash of all the input frames,
+on a per-stream basis. This can be used for equality checks without having
+to do a complete binary comparison.
+
+By default audio frames are converted to signed 16-bit raw audio and
+video frames to raw video before computing the hash, but the output
+of explicit conversions to other codecs can also be used. Timestamps
+are ignored. It uses the SHA-256 cryptographic hash function by default,
+but supports several other algorithms.
+
+The output of the muxer consists of one line per stream of the form:
+@var{streamindex},@var{streamtype},@var{algo}=@var{hash}, where
+@var{streamindex} is the index of the mapped stream, @var{streamtype} is a
+single characer indicating the type of stream, @var{algo} is a short string
+representing the hash function used, and @var{hash} is a hexadecimal number
+representing the computed hash.
+
+@table @option
+@item hash @var{algorithm}
+Use the cryptographic hash function specified by the string @var{algorithm}.
+Supported values include @code{MD5}, @code{murmur3}, @code{RIPEMD128},
+@code{RIPEMD160}, @code{RIPEMD256}, @code{RIPEMD320}, @code{SHA160},
+@code{SHA224}, @code{SHA256} (default), @code{SHA512/224}, @code{SHA512/256},
+@code{SHA384}, @code{SHA512}, @code{CRC32} and @code{adler32}.
+
+@end table
+
+@subsection Examples
+
+To compute the SHA-256 hash of the input converted to raw audio and
+video, and store it in the file @file{out.sha256}:
+@example
+ffmpeg -i INPUT -f streamhash out.sha256
+@end example
+
+To print an MD5 hash to stdout use the command:
+@example
+ffmpeg -i INPUT -f streamhash -hash md5 -
+@end example
+
+See also the @ref{hash} and @ref{framehash} muxers.
+
 @anchor{fifo}
 @section fifo

diff --git a/libavformat/Makefile b/libavformat/Makefile
index efa3a112ae..a61d42b721 100644
--- a/libavformat/Makefile
+++ b/libavformat/Makefile
@@ -494,6 +494,7 @@  OBJS-$(CONFIG_SRT_DEMUXER)               += srtdec.o subtitles.o
 OBJS-$(CONFIG_SRT_MUXER)                 += srtenc.o
 OBJS-$(CONFIG_STL_DEMUXER)               += stldec.o subtitles.o
 OBJS-$(CONFIG_STR_DEMUXER)               += psxstr.o
+OBJS-$(CONFIG_STREAMHASH_MUXER)          += hashenc.o
 OBJS-$(CONFIG_STREAM_SEGMENT_MUXER)      += segment.o
 OBJS-$(CONFIG_SUBVIEWER1_DEMUXER)        += subviewer1dec.o subtitles.o
 OBJS-$(CONFIG_SUBVIEWER_DEMUXER)         += subviewerdec.o subtitles.o
diff --git a/libavformat/allformats.c b/libavformat/allformats.c
index cd00834807..f7fea32b45 100644
--- a/libavformat/allformats.c
+++ b/libavformat/allformats.c
@@ -393,6 +393,7 @@  extern AVInputFormat  ff_srt_demuxer;
 extern AVOutputFormat ff_srt_muxer;
 extern AVInputFormat  ff_str_demuxer;
 extern AVInputFormat  ff_stl_demuxer;
+extern AVOutputFormat ff_streamhash_muxer;
 extern AVInputFormat  ff_subviewer1_demuxer;
 extern AVInputFormat  ff_subviewer_demuxer;
 extern AVInputFormat  ff_sup_demuxer;
diff --git a/libavformat/hashenc.c b/libavformat/hashenc.c
index a9be6d0c81..8e090731ae 100644
--- a/libavformat/hashenc.c
+++ b/libavformat/hashenc.c
@@ -31,6 +31,7 @@  struct HashContext {
     const AVClass *avclass;
     struct AVHashContext **hashes;
     char *hash_name;
+    int per_stream;
     int format_version;
 };

@@ -56,6 +57,13 @@  static const AVOption framehash_options[] = {
 };
 #endif

+#if CONFIG_STREAMHASH_MUXER
+static const AVOption streamhash_options[] = {
+    HASH_OPT("sha256"),
+    { NULL },
+};
+#endif
+
 #if CONFIG_MD5_MUXER
 static const AVOption md5_options[] = {
     HASH_OPT("md5"),
@@ -76,6 +84,7 @@  static int hash_init(struct AVFormatContext *s)
 {
     int res;
     struct HashContext *c = s->priv_data;
+    c->per_stream = 0;
     c->hashes = av_mallocz_array(1, sizeof(c->hashes));
     if (!c->hashes)
         return AVERROR(ENOMEM);
@@ -85,24 +94,66 @@  static int hash_init(struct AVFormatContext *s)
     av_hash_init(c->hashes[0]);
     return 0;
 }
+#endif
+
+#if CONFIG_STREAMHASH_MUXER
+static int streamhash_init(struct AVFormatContext *s)
+{
+    int res, i;
+    struct HashContext *c = s->priv_data;
+    c->per_stream = 1;
+    c->hashes = av_mallocz_array(s->nb_streams, sizeof(c->hashes));
+    if (!c->hashes)
+        return AVERROR(ENOMEM);
+    for (i = 0; i < s->nb_streams; i++) {
+        res = av_hash_alloc(&c->hashes[i], c->hash_name);
+        if (res < 0) {
+            return res;
+        }
+        av_hash_init(c->hashes[i]);
+    }
+    return 0;
+}
+#endif
+
+#if CONFIG_HASH_MUXER || CONFIG_MD5_MUXER || CONFIG_STREAMHASH_MUXER
+static char get_media_type_char(enum AVMediaType type)
+{
+    switch (type) {
+    case AVMEDIA_TYPE_VIDEO:      return 'v';
+    case AVMEDIA_TYPE_AUDIO:      return 'a';
+    case AVMEDIA_TYPE_DATA:       return 'd';
+    case AVMEDIA_TYPE_SUBTITLE:   return 's';
+    case AVMEDIA_TYPE_ATTACHMENT: return 't';
+    default:                      return '?';
+    }
+}

 static int hash_write_packet(struct AVFormatContext *s, AVPacket *pkt)
 {
     struct HashContext *c = s->priv_data;
-    av_hash_update(c->hashes[0], pkt->data, pkt->size);
+    av_hash_update(c->hashes[c->per_stream ? pkt->stream_index : 0], pkt->data, pkt->size);
     return 0;
 }

 static int hash_write_trailer(struct AVFormatContext *s)
 {
     struct HashContext *c = s->priv_data;
-    char buf[AV_HASH_MAX_SIZE*2+128];
-    snprintf(buf, sizeof(buf) - 200, "%s=", av_hash_get_name(c->hashes[0]));
-
-    av_hash_final_hex(c->hashes[0], buf + strlen(buf), sizeof(buf) - strlen(buf));
-    av_strlcatf(buf, sizeof(buf), "\n");
-    avio_write(s->pb, buf, strlen(buf));
-    avio_flush(s->pb);
+    int num_hashes = c->per_stream ? s->nb_streams : 1;
+    for (int i = 0; i < num_hashes; i++) {
+        char buf[AV_HASH_MAX_SIZE*2+128];
+        if (c->per_stream) {
+            AVStream *st = s->streams[i];
+            snprintf(buf, sizeof(buf) - 200, "%d,%c,%s=", i, get_media_type_char(st->codecpar->codec_type),
+                     av_hash_get_name(c->hashes[i]));
+        } else {
+            snprintf(buf, sizeof(buf) - 200, "%s=", av_hash_get_name(c->hashes[i]));
+        }
+        av_hash_final_hex(c->hashes[i], buf + strlen(buf), sizeof(buf) - strlen(buf));
+        av_strlcatf(buf, sizeof(buf), "\n");
+        avio_write(s->pb, buf, strlen(buf));
+        avio_flush(s->pb);
+    }

     return 0;
 }
@@ -110,8 +161,12 @@  static int hash_write_trailer(struct AVFormatContext *s)
 static void hash_free(struct AVFormatContext *s)
 {
     struct HashContext *c = s->priv_data;
-    if (c->hashes)
-        av_hash_freep(&c->hashes[0]);
+    if (c->hashes) {
+        int num_hashes = c->per_stream ? s->nb_streams : 1;
+        for (int i = 0; i < num_hashes; i++) {
+            av_hash_freep(&c->hashes[i]);
+        }
+    }
     av_freep(&c->hashes);
 }
 #endif
@@ -164,6 +219,30 @@  AVOutputFormat ff_md5_muxer = {
 };
 #endif

+#if CONFIG_STREAMHASH_MUXER
+static const AVClass streamhashenc_class = {
+    .class_name = "stream hash muxer",
+    .item_name  = av_default_item_name,
+    .option     = streamhash_options,
+    .version    = LIBAVUTIL_VERSION_INT,
+};
+
+AVOutputFormat ff_streamhash_muxer = {
+    .name              = "streamhash",
+    .long_name         = NULL_IF_CONFIG_SMALL("Per-stream hash testing"),
+    .priv_data_size    = sizeof(struct HashContext),
+    .audio_codec       = AV_CODEC_ID_PCM_S16LE,
+    .video_codec       = AV_CODEC_ID_RAWVIDEO,
+    .init              = streamhash_init,
+    .write_packet      = hash_write_packet,
+    .write_trailer     = hash_write_trailer,
+    .deinit            = hash_free,
+    .flags             = AVFMT_VARIABLE_FPS | AVFMT_TS_NONSTRICT |
+                         AVFMT_TS_NEGATIVE,
+    .priv_class        = &streamhashenc_class,
+};
+#endif
+
 #if CONFIG_FRAMEHASH_MUXER || CONFIG_FRAMEMD5_MUXER
 static void framehash_print_extradata(struct AVFormatContext *s)
 {
@@ -190,6 +269,7 @@  static int framehash_init(struct AVFormatContext *s)
 {
     int res;
     struct HashContext *c = s->priv_data;
+    c->per_stream = 0;
     c->hashes = av_mallocz_array(1, sizeof(c->hashes));
     if (!c->hashes)
         return AVERROR(ENOMEM);
diff --git a/libavformat/version.h b/libavformat/version.h
index edfa73fb97..bcd0408d28 100644
--- a/libavformat/version.h
+++ b/libavformat/version.h
@@ -32,8 +32,8 @@ 
 // Major bumping may affect Ticket5467, 5421, 5451(compatibility with Chromium)
 // Also please add any ticket numbers that you believe might be affected here
 #define LIBAVFORMAT_VERSION_MAJOR  58
-#define LIBAVFORMAT_VERSION_MINOR  32
-#define LIBAVFORMAT_VERSION_MICRO 104
+#define LIBAVFORMAT_VERSION_MINOR  33
+#define LIBAVFORMAT_VERSION_MICRO 100

 #define LIBAVFORMAT_VERSION_INT AV_VERSION_INT(LIBAVFORMAT_VERSION_MAJOR, \
                                                LIBAVFORMAT_VERSION_MINOR, \