| Message ID | 20201013081852.24799-1-yejun.guo@intel.com |
|---|---|
| State | Superseded |
| Headers | show |
| Series | [FFmpeg-devel] dnn_backend_tf.c: add option sess_config for tf backend | expand |
| Context | Check | Description |
|---|---|---|
| andriy/x86_make | success | Make finished |
| andriy/x86_make_fate | success | Make fate finished |
| andriy/PPC64_make | warning | Make failed |
Hi! With proto files they can be serialised to many different formats, is there a reason for it to use hex? The reason I ask is that it's not trivially readable for those debugging the command line. Perhaps something like json is better? Even if not using json, maybe a file is a little easier? In cases where people have a large system using this, it just seems like it would be easier to chase down issues. Thanks, Chris On Tue, Oct 13, 2020, 7:19 PM Guo, Yejun <yejun.guo@intel.com> wrote: > TensorFlow C library accepts config for session options to > set different parameters for the inference. This patch exports > this interface. > > The config is a serialized tensorflow.ConfigProto proto, so we need > two steps to use it: > 1. generate the serialized proto with python (see script example below) > the output looks like: 0xab...cd > where 0xcd is the least significant byte and 0xab is the most significant > byte. > > 2. pass the python script output into ffmpeg with > dnn_processing=options=sess_config=0xab...cd > > The following script is an example to specify one GPU. If the system > contains > 3 GPU cards, the visible_device_list could be '0', '1', '2', '0,1' etc. > '0' does not mean physical GPU card 0, we need to try and see. > And we can also add more opitions here to generate more serialized proto. > > script example to generate serialized proto which specifies one GPU: > import tensorflow as tf > gpu_options = tf.GPUOptions(visible_device_list='0') > config = tf.ConfigProto(gpu_options=gpu_options) > s = config.SerializeToString() > b = ''.join("%02x" % int(ord(b)) for b in s[::-1]) > print('0x%s' % b) > > Signed-off-by: Guo, Yejun <yejun.guo@intel.com> > --- > libavfilter/dnn/dnn_backend_tf.c | 78 +++++++++++++++++++++++++++++--- > 1 file changed, 72 insertions(+), 6 deletions(-) > > diff --git a/libavfilter/dnn/dnn_backend_tf.c > b/libavfilter/dnn/dnn_backend_tf.c > index be860b11b5..7f1b8f0097 100644 > --- a/libavfilter/dnn/dnn_backend_tf.c > +++ b/libavfilter/dnn/dnn_backend_tf.c > @@ -29,14 +29,20 @@ > #include "dnn_backend_native_layer_depth2space.h" > #include "libavformat/avio.h" > #include "libavutil/avassert.h" > +#include "../internal.h" > #include "dnn_backend_native_layer_pad.h" > #include "dnn_backend_native_layer_maximum.h" > #include "dnn_io_proc.h" > > #include <tensorflow/c/c_api.h> > > +typedef struct TFOptions{ > + char *sess_config; > +} TFOptions; > + > typedef struct TFContext { > const AVClass *class; > + TFOptions options; > } TFContext; > > typedef struct TFModel{ > @@ -47,14 +53,15 @@ typedef struct TFModel{ > TF_Status *status; > } TFModel; > > -static const AVClass dnn_tensorflow_class = { > - .class_name = "dnn_tensorflow", > - .item_name = av_default_item_name, > - .option = NULL, > - .version = LIBAVUTIL_VERSION_INT, > - .category = AV_CLASS_CATEGORY_FILTER, > +#define OFFSET(x) offsetof(TFContext, x) > +#define FLAGS AV_OPT_FLAG_FILTERING_PARAM > +static const AVOption dnn_tensorflow_options[] = { > + { "sess_config", "config for SessionOptions", > OFFSET(options.sess_config), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, > FLAGS }, > + { NULL } > }; > > +AVFILTER_DEFINE_CLASS(dnn_tensorflow); > + > static DNNReturnType execute_model_tf(const DNNModel *model, const char > *input_name, AVFrame *in_frame, > const char **output_names, uint32_t > nb_output, AVFrame *out_frame, > int do_ioproc); > @@ -180,10 +187,48 @@ static DNNReturnType load_tf_model(TFModel > *tf_model, const char *model_filename > TF_ImportGraphDefOptions *graph_opts; > TF_SessionOptions *sess_opts; > const TF_Operation *init_op; > + uint8_t *sess_config = NULL; > + int sess_config_length = 0; > + > + // prepare the sess config data > + if (tf_model->ctx.options.sess_config != NULL) { > + char tmp[3]; > + tmp[2] = '\0'; > + > + if (strncmp(tf_model->ctx.options.sess_config, "0x", 2) != 0) { > + av_log(ctx, AV_LOG_ERROR, "sess_config should start with > '0x'\n"); > + return DNN_ERROR; > + } > + > + sess_config_length = strlen(tf_model->ctx.options.sess_config); > + if (sess_config_length % 2 != 0) { > + av_log(ctx, AV_LOG_ERROR, "the length of sess_config is not > even (%s), " > + "please re-generate the config.\n", > + tf_model->ctx.options.sess_config); > + return DNN_ERROR; > + } > + > + sess_config_length -= 2; //ignore the first '0x' > + sess_config_length /= 2; //get the data length in byte > + > + sess_config = av_malloc(sess_config_length); > + if (!sess_config) { > + av_log(ctx, AV_LOG_ERROR, "failed to allocate memory\n"); > + return DNN_ERROR; > + } > + > + for (int i = 0; i < sess_config_length; i++) { > + int index = 2 + (sess_config_length - 1 - i) * 2; > + tmp[0] = tf_model->ctx.options.sess_config[index]; > + tmp[1] = tf_model->ctx.options.sess_config[index + 1]; > + sess_config[i] = strtol(tmp, NULL, 16); > + } > + } > > graph_def = read_graph(model_filename); > if (!graph_def){ > av_log(ctx, AV_LOG_ERROR, "Failed to read model \"%s\" graph\n", > model_filename); > + av_freep(&sess_config); > return DNN_ERROR; > } > tf_model->graph = TF_NewGraph(); > @@ -196,11 +241,23 @@ static DNNReturnType load_tf_model(TFModel > *tf_model, const char *model_filename > TF_DeleteGraph(tf_model->graph); > TF_DeleteStatus(tf_model->status); > av_log(ctx, AV_LOG_ERROR, "Failed to import serialized graph to > model graph\n"); > + av_freep(&sess_config); > return DNN_ERROR; > } > > init_op = TF_GraphOperationByName(tf_model->graph, "init"); > sess_opts = TF_NewSessionOptions(); > + > + if (sess_config) { > + TF_SetConfig(sess_opts, sess_config, > sess_config_length,tf_model->status); > + av_freep(&sess_config); > + if (TF_GetCode(tf_model->status) != TF_OK) { > + av_log(ctx, AV_LOG_ERROR, "Failed to set config for sess > options with %s\n", > + tf_model->ctx.options.sess_config); > + return DNN_ERROR; > + } > + } > + > tf_model->session = TF_NewSession(tf_model->graph, sess_opts, > tf_model->status); > TF_DeleteSessionOptions(sess_opts); > if (TF_GetCode(tf_model->status) != TF_OK) > @@ -595,6 +652,15 @@ DNNModel *ff_dnn_load_model_tf(const char > *model_filename, const char *options, > tf_model->ctx.class = &dnn_tensorflow_class; > tf_model->model = model; > > + //parse options > + av_opt_set_defaults(&tf_model->ctx); > + if (av_opt_set_from_string(&tf_model->ctx, options, NULL, "=", "&") < > 0) { > + av_log(&tf_model->ctx, AV_LOG_ERROR, "Failed to parse options > \"%s\"\n", options); > + av_freep(&tf_model); > + av_freep(&model); > + return NULL; > + } > + > if (load_tf_model(tf_model, model_filename) != DNN_SUCCESS){ > if (load_native_model(tf_model, model_filename) != DNN_SUCCESS){ > av_freep(&tf_model); > -- > 2.17.1 > > _______________________________________________ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
> From: Chris Miceli <chris@miceli.net.au> > Sent: 2020年10月14日 6:15 > To: FFmpeg development discussions and patches <ffmpeg-devel@ffmpeg.org> > Cc: Guo, Yejun <yejun.guo@intel.com> > Subject: Re: [FFmpeg-devel] [PATCH] dnn_backend_tf.c: add option sess_config > for tf backend > > Hi! > > With proto files they can be serialised to many different formats, is there a > reason for it to use hex? The reason I ask is that it's not trivially readable for > those debugging the command line. Perhaps something like json is better? yes, readable message as FFmpeg command line option is very good, but the readable message need to be serialized into proto within ffmpeg since tensorflow C api only accepts the serialized proto, it means that we need to: - add libprotobuf dependency to ffmpeg - know tensorflow's model format so to generate serialized proto. these are not feasible, so ffmpeg has to accept the serialized proto which need to be generated out of ffmpeg. > > Even if not using json, maybe a file is a little easier? In cases where people have a > large system using this, it just seems like it would be easier to chase down issues. yes, serialized proto can be in a file, we can add such option sess_config_file once a customer requires it. As the first step, we can use the command line directly. thanks. > > Thanks, > Chris
diff --git a/libavfilter/dnn/dnn_backend_tf.c b/libavfilter/dnn/dnn_backend_tf.c index be860b11b5..7f1b8f0097 100644 --- a/libavfilter/dnn/dnn_backend_tf.c +++ b/libavfilter/dnn/dnn_backend_tf.c @@ -29,14 +29,20 @@ #include "dnn_backend_native_layer_depth2space.h" #include "libavformat/avio.h" #include "libavutil/avassert.h" +#include "../internal.h" #include "dnn_backend_native_layer_pad.h" #include "dnn_backend_native_layer_maximum.h" #include "dnn_io_proc.h" #include <tensorflow/c/c_api.h> +typedef struct TFOptions{ + char *sess_config; +} TFOptions; + typedef struct TFContext { const AVClass *class; + TFOptions options; } TFContext; typedef struct TFModel{ @@ -47,14 +53,15 @@ typedef struct TFModel{ TF_Status *status; } TFModel; -static const AVClass dnn_tensorflow_class = { - .class_name = "dnn_tensorflow", - .item_name = av_default_item_name, - .option = NULL, - .version = LIBAVUTIL_VERSION_INT, - .category = AV_CLASS_CATEGORY_FILTER, +#define OFFSET(x) offsetof(TFContext, x) +#define FLAGS AV_OPT_FLAG_FILTERING_PARAM +static const AVOption dnn_tensorflow_options[] = { + { "sess_config", "config for SessionOptions", OFFSET(options.sess_config), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS }, + { NULL } }; +AVFILTER_DEFINE_CLASS(dnn_tensorflow); + static DNNReturnType execute_model_tf(const DNNModel *model, const char *input_name, AVFrame *in_frame, const char **output_names, uint32_t nb_output, AVFrame *out_frame, int do_ioproc); @@ -180,10 +187,48 @@ static DNNReturnType load_tf_model(TFModel *tf_model, const char *model_filename TF_ImportGraphDefOptions *graph_opts; TF_SessionOptions *sess_opts; const TF_Operation *init_op; + uint8_t *sess_config = NULL; + int sess_config_length = 0; + + // prepare the sess config data + if (tf_model->ctx.options.sess_config != NULL) { + char tmp[3]; + tmp[2] = '\0'; + + if (strncmp(tf_model->ctx.options.sess_config, "0x", 2) != 0) { + av_log(ctx, AV_LOG_ERROR, "sess_config should start with '0x'\n"); + return DNN_ERROR; + } + + sess_config_length = strlen(tf_model->ctx.options.sess_config); + if (sess_config_length % 2 != 0) { + av_log(ctx, AV_LOG_ERROR, "the length of sess_config is not even (%s), " + "please re-generate the config.\n", + tf_model->ctx.options.sess_config); + return DNN_ERROR; + } + + sess_config_length -= 2; //ignore the first '0x' + sess_config_length /= 2; //get the data length in byte + + sess_config = av_malloc(sess_config_length); + if (!sess_config) { + av_log(ctx, AV_LOG_ERROR, "failed to allocate memory\n"); + return DNN_ERROR; + } + + for (int i = 0; i < sess_config_length; i++) { + int index = 2 + (sess_config_length - 1 - i) * 2; + tmp[0] = tf_model->ctx.options.sess_config[index]; + tmp[1] = tf_model->ctx.options.sess_config[index + 1]; + sess_config[i] = strtol(tmp, NULL, 16); + } + } graph_def = read_graph(model_filename); if (!graph_def){ av_log(ctx, AV_LOG_ERROR, "Failed to read model \"%s\" graph\n", model_filename); + av_freep(&sess_config); return DNN_ERROR; } tf_model->graph = TF_NewGraph(); @@ -196,11 +241,23 @@ static DNNReturnType load_tf_model(TFModel *tf_model, const char *model_filename TF_DeleteGraph(tf_model->graph); TF_DeleteStatus(tf_model->status); av_log(ctx, AV_LOG_ERROR, "Failed to import serialized graph to model graph\n"); + av_freep(&sess_config); return DNN_ERROR; } init_op = TF_GraphOperationByName(tf_model->graph, "init"); sess_opts = TF_NewSessionOptions(); + + if (sess_config) { + TF_SetConfig(sess_opts, sess_config, sess_config_length,tf_model->status); + av_freep(&sess_config); + if (TF_GetCode(tf_model->status) != TF_OK) { + av_log(ctx, AV_LOG_ERROR, "Failed to set config for sess options with %s\n", + tf_model->ctx.options.sess_config); + return DNN_ERROR; + } + } + tf_model->session = TF_NewSession(tf_model->graph, sess_opts, tf_model->status); TF_DeleteSessionOptions(sess_opts); if (TF_GetCode(tf_model->status) != TF_OK) @@ -595,6 +652,15 @@ DNNModel *ff_dnn_load_model_tf(const char *model_filename, const char *options, tf_model->ctx.class = &dnn_tensorflow_class; tf_model->model = model; + //parse options + av_opt_set_defaults(&tf_model->ctx); + if (av_opt_set_from_string(&tf_model->ctx, options, NULL, "=", "&") < 0) { + av_log(&tf_model->ctx, AV_LOG_ERROR, "Failed to parse options \"%s\"\n", options); + av_freep(&tf_model); + av_freep(&model); + return NULL; + } + if (load_tf_model(tf_model, model_filename) != DNN_SUCCESS){ if (load_native_model(tf_model, model_filename) != DNN_SUCCESS){ av_freep(&tf_model);
TensorFlow C library accepts config for session options to set different parameters for the inference. This patch exports this interface. The config is a serialized tensorflow.ConfigProto proto, so we need two steps to use it: 1. generate the serialized proto with python (see script example below) the output looks like: 0xab...cd where 0xcd is the least significant byte and 0xab is the most significant byte. 2. pass the python script output into ffmpeg with dnn_processing=options=sess_config=0xab...cd The following script is an example to specify one GPU. If the system contains 3 GPU cards, the visible_device_list could be '0', '1', '2', '0,1' etc. '0' does not mean physical GPU card 0, we need to try and see. And we can also add more opitions here to generate more serialized proto. script example to generate serialized proto which specifies one GPU: import tensorflow as tf gpu_options = tf.GPUOptions(visible_device_list='0') config = tf.ConfigProto(gpu_options=gpu_options) s = config.SerializeToString() b = ''.join("%02x" % int(ord(b)) for b in s[::-1]) print('0x%s' % b) Signed-off-by: Guo, Yejun <yejun.guo@intel.com> --- libavfilter/dnn/dnn_backend_tf.c | 78 +++++++++++++++++++++++++++++--- 1 file changed, 72 insertions(+), 6 deletions(-)