[FFmpeg-devel,v2,4/5] avutil/channel_layout: add av_channel_layout_retype()

diff --git a/doc/APIchanges b/doc/APIchanges
index cdb9b6a458..221fea30c2 100644
--- a/doc/APIchanges
+++ b/doc/APIchanges
@@ -2,6 +2,9 @@  The last version increases of all libraries were on 2023-02-09
 
 API changes, most recent first:
 
+2024-02-xx - xxxxxxxxxx - lavu 58.38.100 - channel_layout.h
+  Add av_channel_layout_retype().
+
 2024-02-xx - xxxxxxxxxx - lavu 58.37.100 - channel_layout.h
   Add av_channel_layout_custom_init().
 
diff --git a/libavutil/channel_layout.c b/libavutil/channel_layout.c
index 40e31e9d12..7f51c076dc 100644
--- a/libavutil/channel_layout.c
+++ b/libavutil/channel_layout.c
@@ -1036,3 +1036,109 @@  uint64_t av_channel_layout_subset(const AVChannelLayout *channel_layout,
 
     return ret;
 }
+
+static int64_t masked_description(AVChannelLayout *channel_layout, int start_channel)
+{
+    uint64_t mask = 0;
+    for (int i = start_channel; i < channel_layout->nb_channels; i++) {
+        enum AVChannel ch = channel_layout->u.map[i].id;
+        if (ch >= 0 && ch < 63 && mask < (1ULL << ch))
+            mask |= (1ULL << ch);
+        else
+            return AVERROR(EINVAL);
+    }
+    return mask;
+}
+
+static int has_channel_names(AVChannelLayout *channel_layout)
+{
+    if (channel_layout->order != AV_CHANNEL_ORDER_CUSTOM)
+        return 0;
+    for (int i = 0; i < channel_layout->nb_channels; i++)
+        if (channel_layout->u.map[i].name[0])
+            return 1;
+    return 0;
+}
+
+int av_channel_layout_retype(AVChannelLayout *channel_layout, enum AVChannelOrder order, int flags)
+{
+    int allow_lossy = !(flags & AV_CHANNEL_LAYOUT_RETYPE_FLAG_LOSSLESS);
+    int lossy;
+
+    if (!av_channel_layout_check(channel_layout))
+        return AVERROR(EINVAL);
+
+    if (channel_layout->order == order)
+        return 0;
+
+    switch (order) {
+    case AV_CHANNEL_ORDER_UNSPEC: {
+        int nb_channels = channel_layout->nb_channels;
+        if (channel_layout->order == AV_CHANNEL_ORDER_CUSTOM) {
+            lossy = 0;
+            for (int i = 0; i < nb_channels; i++) {
+                if (channel_layout->u.map[i].id != AV_CHAN_UNKNOWN || channel_layout->u.map[i].name[0]) {
+                    lossy = 1;
+                    break;
+                }
+            }
+        } else {
+            lossy = 1;
+        }
+        if (!lossy || allow_lossy) {
+            av_channel_layout_uninit(channel_layout);
+            channel_layout->order       = AV_CHANNEL_ORDER_UNSPEC;
+            channel_layout->nb_channels = nb_channels;
+            return lossy;
+        }
+        return AVERROR(ENOSYS);
+        }
+    case AV_CHANNEL_ORDER_NATIVE:
+        if (channel_layout->order == AV_CHANNEL_ORDER_CUSTOM) {
+            int64_t mask = masked_description(channel_layout, 0);
+            if (mask < 0)
+                return AVERROR(ENOSYS);
+            lossy = has_channel_names(channel_layout);
+            if (!lossy || allow_lossy) {
+                av_channel_layout_uninit(channel_layout);
+                av_channel_layout_from_mask(channel_layout, mask);
+                return lossy;
+            }
+        }
+        return AVERROR(ENOSYS);
+    case AV_CHANNEL_ORDER_CUSTOM: {
+        AVChannelLayout custom = { 0 };
+        int ret = av_channel_layout_custom_init(&custom, channel_layout->nb_channels);
+        if (ret < 0)
+            return ret;
+        if (channel_layout->order != AV_CHANNEL_ORDER_UNSPEC)
+            for (int i = 0; i < channel_layout->nb_channels; i++)
+                custom.u.map[i].id = av_channel_layout_channel_from_index(channel_layout, i);
+        av_channel_layout_uninit(channel_layout);
+        *channel_layout = custom;
+        return 0;
+        }
+    case AV_CHANNEL_ORDER_AMBISONIC:
+        if (channel_layout->order == AV_CHANNEL_ORDER_CUSTOM) {
+            int64_t mask;
+            int nb_channels = channel_layout->nb_channels;
+            int order = ambisonic_order(channel_layout);
+            if (order < 0)
+                return AVERROR(ENOSYS);
+            mask = masked_description(channel_layout, (order + 1) * (order + 1));
+            if (mask < 0)
+                return AVERROR(ENOSYS);
+            lossy = has_channel_names(channel_layout);
+            if (!lossy || allow_lossy) {
+                av_channel_layout_uninit(channel_layout);
+                channel_layout->order       = AV_CHANNEL_ORDER_AMBISONIC;
+                channel_layout->nb_channels = nb_channels;
+                channel_layout->u.mask      = mask;
+                return lossy;
+            }
+        }
+        return AVERROR(ENOSYS);
+    default:
+        return AVERROR(EINVAL);
+    }
+}
diff --git a/libavutil/channel_layout.h b/libavutil/channel_layout.h
index dcc320cbfe..10260d3ec5 100644
--- a/libavutil/channel_layout.h
+++ b/libavutil/channel_layout.h
@@ -821,6 +821,46 @@  int av_channel_layout_check(const AVChannelLayout *channel_layout);
  */
 int av_channel_layout_compare(const AVChannelLayout *chl, const AVChannelLayout *chl1);
 
+/**
+ * The conversion must be lossless.
+ */
+#define AV_CHANNEL_LAYOUT_RETYPE_FLAG_LOSSLESS (1 << 0)
+
+/**
+ * Change the AVChannelOrder of a channel layout.
+ *
+ * Change of AVChannelOrder can be either lossless or lossy. In case of a
+ * lossless conversion all the channel designations and the associated channel
+ * names (if any) are kept. On a lossy conversion the channel names and channel
+ * designations might be lost depending on the capabilities of the desired
+ * AVChannelOrder. Note that some conversions are simply not possible in which
+ * case this function returns AVERROR(ENOSYS).
+ *
+ * The following conversions are supported:
+ *
+ * Any       -> Custom     : Always possible, always lossless.
+ * Any       -> Unspecified: Always possible, lossless if channel designations
+ *   are all unknown and channel names are not used, lossy otherwise.
+ * Custom    -> Ambisonic  : Possible if it contains ambisonic channels with
+ *   optional non-diegetic channels in the end. Lossy if the channels have
+ *   custom names, lossless otherwise.
+ * Custom    -> Native     : Possible if it contains native channels in native
+ *     order. Lossy if the channels have custom names, lossless otherwise.
+ *
+ * On error this function keeps the original channel layout untouched.
+ *
+ * @param channel_layout channel layout which will be changed
+ * @param order the desired channel layout order
+ * @param flags a combination of AV_CHANNEL_LAYOUT_RETYPE_FLAG_* constants
+ * @return 0 if the conversion was successful and lossless or if the channel
+ *           layout was already in the desired order
+ *         1 if the conversion was successful but lossy
+ *         AVERROR(ENOSYS) if the conversion was not possible (or would be
+ *           lossy and AV_CHANNEL_LAYOUT_RETYPE_FLAG_LOSSLESS was specified)
+ *         AVERROR(EINVAL), AVERROR(ENOMEM) on error
+ */
+int av_channel_layout_retype(AVChannelLayout *channel_layout, enum AVChannelOrder order, int flags);
+
 /**
  * @}
  */
diff --git a/libavutil/version.h b/libavutil/version.h
index 3b38f8f5da..cebf4a0acd 100644
--- a/libavutil/version.h
+++ b/libavutil/version.h
@@ -79,7 +79,7 @@ 
  */
 
 #define LIBAVUTIL_VERSION_MAJOR  58
-#define LIBAVUTIL_VERSION_MINOR  37
+#define LIBAVUTIL_VERSION_MINOR  38
 #define LIBAVUTIL_VERSION_MICRO 100
 
 #define LIBAVUTIL_VERSION_INT   AV_VERSION_INT(LIBAVUTIL_VERSION_MAJOR, \