Not able to decode mp4 file using latest ffmpeg library : av_decode_video2 - c

I am writing a wrapper code around latest ffmpeg library. I am supplying MP4 files from local system. My problem is that I am unable to get any decoded frames when I use av_decode_video2(). The return value comes out to be negative. I have used av_read_frame() which returns 0. I googled about the problem I am facing but no where could I find the correct explanation. Please give me insight here. Pasting the pseudo code here.
av_init_packet(avpkt);
picture=av_frame_alloc();
pFrameRGB=av_frame_alloc();
codec = avcodec_find_decoder(CODEC_ID_H264);
c= avcodec_alloc_context3(codec)
avcodec_open2(decoderLibraryData->c, decoderLibraryData->codec, NULL)
FormatContext = avformat_alloc_context();
char *pUrl ="./1.MP4";
iRet = avformat_open_input(atContext, pUrl, pFmt, NULL);
if(FormatContext == NULL)
{
printf("could not assign any memory !!!!!!!!! \n");
}
avformat_find_stream_info(FormatContext, NULL);
while(av_read_frame(FormatContext,avpkt) >= 0)
{
len = avcodec_decode_video2(c, picture, &got_picture,avpkt);
printf("CODEC MANAGER len %d Frame decompressed %d \n",len,got_picture);
if (len <= 0)
{
return ERROR;
}
}
}
if(lastHeight != 0 && lastWidth != 0)
{
if(lastWidth != c->width || lastHeight != c->height )
{
av_free(buffer);
buffer = NULL;
lastWidth = c->width;
lastHeight = c->height;
}
}
else
{
lastWidth = c->width;
lastHeight = c->height;
}
decodeFlag = 1;
if(!buffer)
{
int numBytes;
v_mutex_lock(globalCodecLock);
switch(inPixFormat)
{
case RGB:
// Determine required buffer size and allocate buffer
numBytes=avpicture_get_size(PIX_FMT_RGB24, c->width, c->height);
buffer=(uint8_t *)av_malloc(numBytes*sizeof(uint8_t));
avpicture_fill((AVPicture *)pFrameRGB,buffer,PIX_FMT_RGB24,c->width, c->height);
if(cntxt)
sws_freeContext(cntxt);
cntxt = sws_getContext(c->width, c->height, c->pix_fmt,
c->width, c->height, PIX_FMT_RGB24, SWS_BICUBIC, NULL, NULL, NULL);
break;
}
v_mutex_unlock(globalCodecLock);
if(cntxt == NULL)
{
printf("sws_getContext error\n");
return ERROR;
}
}
{
sws_scale(cntxt, picture->data, picture->linesize, 0, c->height, pFrameRGB->data, pFrameRGB->linesize);
if(rgbBuff)
{
if(c->width <= *width && c->height <= *height)
{
saveFrame(pFrameRGB, c->width, c->height, rgbBuff,inPixFormat);
*width = c->width;
*height = c->height;
rs = SUCCESS;
break;
}
else
{
rs = VA_LOWBUFFERSIZE;
}
}
else
{
rs = VA_LOWBUFFERSIZE;
}
}
if(width)
{
*width = c->width;
}
if(height)
{
*height = c->height;
}
if(rs == VA_LOWBUFFERSIZE)
{
break;
}
I am getting the return value of av_read_frame as 0 but av_decode_video2 returns value in negative. I am not able to get any clue here.

Make sure you have called
av_register_all();
or
avcodec_register_all();
at the beginning of your app.
Also it seems the problem is from calling avformat_find_stream_info. Test with the following code:
AVPacket avpkt;
av_init_packet(&avpkt);
AVFrame* picture = av_frame_alloc();
AVFrame* pFrameRGB = av_frame_alloc();
AVFormatContext* c2 = avformat_alloc_context();
char *pUrl = "C:/Sample Videos/20-06-34.MP4";
int video_stream_index = 0;
AVInputFormat* pFmt;
int iRet = avformat_open_input(&c2, pUrl, pFmt, NULL);
AVStream* stream = c2->streams[video_stream_index];
AVCodec* codec = avcodec_find_decoder(stream->codec->codec_id);
avcodec_open2(stream->codec, codec, NULL);
if (c2 == NULL)
{
printf("could not assign any memory !!!!!!!!! \n");
}
while (av_read_frame(c2, &avpkt) >= 0)
{
int got_picture;
int len = avcodec_decode_video2(stream->codec, picture, &got_picture, &avpkt);
printf("CODEC MANAGER len %d Frame decompressed %d \n", len, got_picture);
if (len <= 0)
{
return ERROR;
}
}

Related

H264 codec encode, decode and write to file

I try to use ffmpeg and h264 codec to translate the video in realtime. But at the state of decoding encoded frame, I get some "bad" image.
Init encoder and decoder:
VCSession *vc_new_x264(Logger *log, ToxAV *av, uint32_t friend_number, toxav_video_receive_frame_cb *cb, void *cb_data,
VCSession *vc)
{
// ------ ffmpeg encoder ------
AVCodec *codec2 = NULL;
vc->h264_encoder_ctx = NULL;//AVCodecContext type
codec2 = NULL;
avcodec_register_all();
codec2 = avcodec_find_encoder(AV_CODEC_ID_H264);
if (codec2 == NULL)
{
LOGGER_WARNING(log, "h264: not find encoder");
}
vc->h264_encoder_ctx = avcodec_alloc_context3(codec2);
vc->h264_out_pic2 = av_packet_alloc();
vc->h264_encoder_ctx->bit_rate = 10 *1000 * 1000;
vc->h264_encoder_ctx->width = 800;
vc->h264_encoder_ctx->height = 600;
vc->h264_enc_width = vc->h264_encoder_ctx->width;
vc->h264_enc_height = vc->h264_encoder_ctx->height;
vc->h264_encoder_ctx->time_base = (AVRational) {
1, 30
};
vc->h264_encoder_ctx->gop_size = 30;
vc->h264_encoder_ctx->max_b_frames = 1;
vc->h264_encoder_ctx->pix_fmt = AV_PIX_FMT_YUV420P;
av_opt_set(vc->h264_encoder_ctx->priv_data, "preset", "veryfast", 0);
av_opt_set(vc->h264_encoder_ctx->priv_data, "annex_b", "1", 0);
av_opt_set(vc->h264_encoder_ctx->priv_data, "repeat_headers", "1", 0);
av_opt_set(vc->h264_encoder_ctx->priv_data, "tune", "zerolatency", 0);
av_opt_set_int(vc->h264_encoder_ctx->priv_data, "zerolatency", 1, 0);
vc->h264_encoder_ctx->time_base.num = 1;
vc->h264_encoder_ctx->time_base.den = 1000;
vc->h264_encoder_ctx->framerate = (AVRational) {
1000, 40
};
AVDictionary *opts = NULL;
if (avcodec_open2(vc->h264_encoder_ctx, codec2, &opts) < 0) {
LOGGER_ERROR(log, "could not open codec H264 on encoder");
}
av_dict_free(&opts);
AVCodec *codec = NULL;
vc->h264_decoder_ctx = NULL;// AVCodecContext - type
codec = NULL;
codec = avcodec_find_decoder(AV_CODEC_ID_H264);
if (!codec) {
LOGGER_WARNING(log, "codec not found H264 on decoder");
}
vc->h264_decoder_ctx = avcodec_alloc_context3(codec);
if (codec->capabilities & AV_CODEC_CAP_TRUNCATED) {
vc->h264_decoder_ctx->flags |= AV_CODEC_FLAG_TRUNCATED; /* we do not send complete frames */
}
if (codec->capabilities & AV_CODEC_FLAG_LOW_DELAY) {
vc->h264_decoder_ctx->flags |= AV_CODEC_FLAG_LOW_DELAY;
}
vc->h264_decoder_ctx->flags |= AV_CODEC_FLAG2_SHOW_ALL;
vc->h264_decoder_ctx->refcounted_frames = 0;
vc->h264_decoder_ctx->delay = 0;
vc->h264_decoder_ctx->sw_pix_fmt = AV_PIX_FMT_YUV420P;
av_opt_set_int(vc->h264_decoder_ctx->priv_data, "delay", 0, AV_OPT_SEARCH_CHILDREN);
vc->h264_decoder_ctx->time_base = (AVRational) {
40, 1000
};
vc->h264_decoder_ctx->framerate = (AVRational) {
1000, 40
};
if (avcodec_open2(vc->h264_decoder_ctx, codec, NULL) < 0) {
LOGGER_WARNING(log, "could not open codec H264 on decoder");
}
vc->h264_decoder_ctx->refcounted_frames = 0;
return vc;
}
Encoding (in this function i encode frame and for debugging decode and save him in file):
uint32_t encode_frame_h264_p(ToxAV *av, uint32_t friend_number, uint16_t width, uint16_t height,
const uint8_t *y,
const uint8_t *u, const uint8_t *v, ToxAVCall *call,
uint64_t *video_frame_record_timestamp,
int vpx_encode_flags,
x264_nal_t **nal,
int *i_frame_size)
{
AVFrame *frame;
int ret;
uint32_t result = 1;
frame = av_frame_alloc();
frame->format = call->video->h264_encoder_ctx->pix_fmt;
frame->width = width;
frame->height = height;
ret = av_frame_get_buffer(frame, 32);
if (ret < 0) {
LOGGER_ERROR(av->m->log, "av_frame_get_buffer:Could not allocate the video frame data");
}
/* make sure the frame data is writable */
ret = av_frame_make_writable(frame);
if (ret < 0) {
LOGGER_ERROR(av->m->log, "av_frame_make_writable:ERROR");
}
frame->pts = (int64_t)(*video_frame_record_timestamp);
// copy YUV frame data into buffers
memcpy(frame->data[0], y, width * height);
memcpy(frame->data[1], u, (width / 2) * (height / 2));
memcpy(frame->data[2], v, (width / 2) * (height / 2));
// encode the frame
ret = avcodec_send_frame(call->video->h264_encoder_ctx, frame);
if (ret < 0) {
LOGGER_ERROR(av->m->log, "Error sending a frame for encoding:ERROR");
}
ret = avcodec_receive_packet(call->video->h264_encoder_ctx, call->video->h264_out_pic2);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
*i_frame_size = 0;
} else if (ret < 0) {
*i_frame_size = 0;
// fprintf(stderr, "Error during encoding\n");
} else {
// Decoded encoded frame and save him to file
saveInFile(call->video->h264_decoder_ctx, frame, call->video->h264_out_pic2, "/home/user/testSave");
// printf("Write packet %3"PRId64" (size=%5d)\n", call->video->h264_out_pic2->pts, call->video->h264_out_pic2->size);
// fwrite(call->video->h264_out_pic2->data, 1, call->video->h264_out_pic2->size, outfile);
global_encoder_delay_counter++;
if (global_encoder_delay_counter > 60) {
global_encoder_delay_counter = 0;
LOGGER_DEBUG(av->m->log, "enc:delay=%ld",
(long int)(frame->pts - (int64_t)call->video->h264_out_pic2->pts)
);
}
*i_frame_size = call->video->h264_out_pic2->size;
*video_frame_record_timestamp = (uint64_t)call->video->h264_out_pic2->pts;
result = 0;
}
av_frame_free(&frame);
return result;
}
Decode and save frame code:
void saveInFile(AVCodecContext *dec_ctx, AVFrame *frame, AVPacket *pkt, const char *filename)
{
if (!pkt)
return;
char buf[1024];
int ret;
static int curNumber = 0;
ret = avcodec_send_packet(dec_ctx, pkt);
if (ret < 0 && ret != AVERROR_EOF)
{
fprintf(stderr, "Error sending a packet for decoding'\n");
if ( ret == AVERROR(EAGAIN))
return;
if (ret == AVERROR(EINVAL))
return;
if (ret == AVERROR(ENOMEM))
return;
}
ret = avcodec_receive_frame(dec_ctx, frame);
if (ret == AVERROR(EAGAIN) )
return;
if (ret == AVERROR_EOF)
{
return;
}
else if (ret < 0)
{
fprintf(stderr, "Error during decoding\n");
}
printf("saving frame %3d\n", dec_ctx->frame_number);
sprintf(buf, "%s%d", filename, curNumber);
curNumber++;
pgm_save(frame->data[0], frame->linesize[0], frame->width, frame->height, buf);
}
void pgm_save(unsigned char* buf, int wrap, int xsize, int ysize, char *filename)
{
FILE *f;
int i;
f = fopen(filename, "w");
fprintf(f, "P5\n%d %d\n%d\n", xsize, ysize, 255);
for (i =0; i < ysize; i++)
fwrite(buf + i* wrap, 1, xsize, f);
fclose(f);
}
After this manipulation I have smth like that:

How to set l2tp preshared key?

I need to create RASENTRY for L2TP with pre-shared key set. So far, I can see that entry has somewhat correct flags, but no key is set, unfortunately.
Here is code:
int common_ras_manager_create_entry(const char* server_address, const char* username, const char* password, MY_VPN_CONNECTION_TYPE connection_type, const char* preshared_key)
{
DWORD EntryInfoSize = 0;
DWORD DeviceInfoSize = 0;
DWORD Ret;
LPRASENTRY lpRasEntry;
LPBYTE lpDeviceInfo;
// Get buffer sizing information for a default phonebook entry
if ((Ret = RasGetEntryProperties(NULL, "", NULL, &EntryInfoSize, lpDeviceInfo, &DeviceInfoSize)) != 0)
{
if (Ret != ERROR_BUFFER_TOO_SMALL)
{
printf("RasGetEntryProperties sizing failed with error %d\n", Ret);
return Ret;
}
}
lpRasEntry = (LPRASENTRY) GlobalAlloc(GPTR, EntryInfoSize);
if (DeviceInfoSize == 0)
lpDeviceInfo = NULL;
else
lpDeviceInfo = (LPBYTE) GlobalAlloc(GPTR, DeviceInfoSize);
// Get default phonebook entry
lpRasEntry->dwSize = sizeof(RASENTRY);
if ((Ret = RasGetEntryProperties(NULL, "", lpRasEntry, &EntryInfoSize, lpDeviceInfo, &DeviceInfoSize)) != 0)
{
printf("RasGetEntryProperties failed with error %d\n", Ret);
return Ret;
}
// Validate new phonebook name "Testentry"
if ((Ret = RasValidateEntryName(NULL, APP_NAME)) != ERROR_SUCCESS)
{
printf("RasValidateEntryName failed with error %d\n", Ret);
if (Ret != ERROR_ALREADY_EXISTS)
return Ret;
}
LPRASDEVINFO ras_devices;
DWORD cb =sizeof(RASDEVINFO);
DWORD cbDevices = 0;
ras_devices = (LPRASDEVINFO)HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, cb);
if (NULL == ras_devices)
{
printf("HeapAlloc failed.\n");
return ERROR_OUTOFMEMORY;
}
ras_devices->dwSize = sizeof(RASDEVINFO);
if ((Ret = RasEnumDevices(ras_devices, &cb, &cbDevices)) != ERROR_SUCCESS)
{
printf("RasEnumDevices failed with error %d\n", Ret);
switch(Ret)
{
case ERROR_BUFFER_TOO_SMALL:
printf("buffer too small");
HeapFree(GetProcessHeap(), 0, (LPVOID)ras_devices);
ras_devices = (LPRASDEVINFO)HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, cb);
if (NULL == ras_devices)
{
printf("HeapAlloc failed.\n");
return ERROR_OUTOFMEMORY;
}
ras_devices->dwSize = sizeof(RASDEVINFO);
Ret = RasEnumDevices(ras_devices, &cb, &cbDevices);
if (ERROR_SUCCESS == Ret)
{
//fSuccess = TRUE;
}
else
{
printf("RasEnumDevices failed again: Error = %d\n", Ret);
return Ret;
//goto done;
}
break;
case ERROR_NOT_ENOUGH_MEMORY:
printf("ERROR_NOT_ENOUGH_MEMORY");
return Ret;
break;
case ERROR_INVALID_PARAMETER:
printf("ERROR_INVALID_PARAMETER");
return Ret;
break;
case ERROR_INVALID_USER_BUFFER:
printf("ERROR_INVALID_USER_BUFFER");
return Ret;
break;
}
}
DWORD dwVpnStrategy = 0;
char device_name_mask[5];
strcpy(device_name_mask, "");
gboolean preshared_key_valid = 0;
switch(connection_type)
{
case PPTP:
strcpy(device_name_mask, "PPTP");
dwVpnStrategy = VS_PptpOnly;
break;
case L2TP:
if (preshared_key == 0 || strlen(preshared_key) == 0)
{
printf("CRITICAL: preshared key not set.");
return 1;
}
else
{
preshared_key_valid = TRUE;
}
strcpy(device_name_mask, "L2TP");
dwVpnStrategy = VS_L2tpOnly;
break;
}
int i =0;
for (i = 0; i < cbDevices; i++)
{
RASDEVINFO r = ras_devices[i];
if (strstr(r.szDeviceName, device_name_mask))
{
break;
}
}
//lpRasEntry->dwfOptions |= RASEO_SpecificIpAddr;
//lpRasEntry->szLocalPhoneNumber = RASDT_Vpn;
lpRasEntry->dwfNetProtocols |= RASNP_Ip;
lpRasEntry->dwFramingProtocol = RASFP_Ppp;
lstrcpy(lpRasEntry->szDeviceType, RASDT_Vpn);
lstrcpy(lpRasEntry->szDeviceName, ras_devices[i].szDeviceName);
lstrcpy(lpRasEntry->szLocalPhoneNumber, server_address);
lpRasEntry->dwVpnStrategy = dwVpnStrategy; // VS_PptpOnly; VS_SstpOnly
if (preshared_key_valid)
{
L2TP_CONFIG_DATA* data = GlobalAlloc(GPTR, sizeof(L2TP_CONFIG_DATA));
lpRasEntry->dwfOptions2 |= RASEO2_UsePreSharedKey;
data->dwOffsetKey = 16;
memcpy((PBYTE)data + data->dwOffsetKey, preshared_key, strlen(preshared_key));
data->dwAuthType =L2TP_IPSEC_AUTH_PRESHAREDKEY;
RasSetCustomAuthData(NULL, APP_NAME, data, sizeof(L2TP_CONFIG_DATA));
}
if ((Ret = RasSetEntryProperties(NULL, APP_NAME, lpRasEntry, EntryInfoSize, lpDeviceInfo, DeviceInfoSize)) != 0)
{
printf("RasSetEntryProperties failed with error %d\n", Ret);
return Ret;
}
//if ((Ret = RasSetCredentials(NULL, lpRasEntry.))
}
I cant find where is buffer to fill for pre-shared key.
Following code works fine.
// l2tp
if (preshared_key_valid)
{
RASCREDENTIALS ras_cre_psk = {0};
ras_cre_psk.dwSize = sizeof(ras_cre_psk);
ras_cre_psk.dwMask = 0x00000010; //RASCM_PreSharedKey;
wcscpy(ras_cre_psk.szPassword, preshared_key);
if ((Ret = RasSetCredentials(NULL, APP_NAME, &ras_cre_psk, FALSE)))
{
printf("RasSetCredentials failed with error %d\n", Ret);
return Ret;
}
}

FFmpeg C API - syncing video and audio

I am trimming video and having a hard getting the audio to sync correctly. The code below is as close as I've gotten it work. I've tried both re-encoding and not re-encoding the output streams.
The video trims correctly and is written to the output container. The audio stream also trims correctly, but is written to the front of the output container. For example if the trim length is 10s - the correct portion of audio plays for 10s and then the correct portion of video plays.
//////// audio stream ////////
const AVStream *input_stream_audio = input_container->streams[audio_stream_index];
const AVCodec *decoder_audio = avcodec_find_decoder(input_stream_audio->codec->codec_id);
if(!decoder_audio) {
cleanup(decoded_packet, output_container, decoded_frame);
avformat_close_input(&input_container);
LOGE("=> Audio decoder not found");
return -1;
}
if(avcodec_open2(input_stream_audio->codec, decoder_audio, NULL) < 0) {
cleanup(decoded_packet, output_container, decoded_frame);
avformat_close_input(&input_container);
LOGE("=> Error opening audio decoder");
return -1;
}
AVStream *output_stream_audio = avformat_new_stream(output_container, NULL);
if(avcodec_copy_context(output_stream_audio->codec, input_stream_audio->codec) != 0){
LOGE("=> Failed to Copy audio Context ");
return -1;
}
else {
LOGI("=> Copied audio context ");
output_stream_audio->codec->codec_id = input_stream_audio->codec->codec_id;
output_stream_audio->codec->codec_tag = 0;
output_stream_audio->pts = input_stream_audio->pts;
output_stream_audio->time_base.num = input_stream_audio->time_base.num;
output_stream_audio->time_base.den = input_stream_audio->time_base.den;
}
if(avio_open(&output_container->pb, output_file, AVIO_FLAG_WRITE) < 0) {
cleanup(decoded_packet, output_container, decoded_frame);
avformat_close_input(&input_container);
LOGE("=> Error opening output file");
return -1;
}
// allocate frame for conversion
decoded_frame = avcodec_alloc_frame();
if(!decoded_frame) {
cleanup(decoded_packet, output_container, decoded_frame);
avformat_close_input(&input_container);
LOGE("=> Error allocating frame");
return -1;
}
av_dump_format(input_container, 0, input_file, 0);
avformat_write_header(output_container, NULL);
av_init_packet(&decoded_packet);
decoded_packet.data = NULL;
decoded_packet.size = 0;
int current_frame_num = 1;
int current_frame_num_audio = 1;
int got_frame, len;
AVRational default_timebase;
default_timebase.num = 1;
default_timebase.den = AV_TIME_BASE;
int64_t starttime_int64 = av_rescale_q((int64_t)( 12.0 * AV_TIME_BASE ), AV_TIME_BASE_Q, input_stream->time_base);
int64_t endtime_int64 = av_rescale_q((int64_t)( 18.0 * AV_TIME_BASE ), AV_TIME_BASE_Q, input_stream->time_base);
LOGI("=> starttime_int64: %" PRId64, starttime_int64);
LOGI("=> endtime_int64: %" PRId64, endtime_int64);
int64_t starttime_int64_audio = av_rescale_q((int64_t)( 12.0 * AV_TIME_BASE ), AV_TIME_BASE_Q, input_stream_audio->time_base);
int64_t endtime_int64_audio = av_rescale_q((int64_t)( 18.0 * AV_TIME_BASE ), AV_TIME_BASE_Q, input_stream_audio->time_base);
LOGI("=> starttime_int64_audio: %" PRId64, starttime_int64_audio);
LOGI("=> endtime_int64_audio: %" PRId64, endtime_int64_audio);
// loop input container and decode frames
while(av_read_frame(input_container, &decoded_packet)>=0) {
// video packets
if (decoded_packet.stream_index == video_stream_index) {
len = avcodec_decode_video2(input_stream->codec, decoded_frame, &got_frame, &decoded_packet);
if(len < 0) {
cleanup(decoded_packet, output_container, decoded_frame);
avformat_close_input(&input_container);
LOGE("=> No frames to decode");
return -1;
}
// this is the trim range we're looking for
if(got_frame && decoded_frame->pkt_pts >= starttime_int64 && decoded_frame->pkt_pts <= endtime_int64) {
av_init_packet(&encoded_packet);
encoded_packet.data = NULL;
encoded_packet.size = 0;
ret = avcodec_encode_video2(output_stream->codec, &encoded_packet, decoded_frame, &got_frame);
if (ret < 0) {
cleanup(decoded_packet, output_container, decoded_frame);
avformat_close_input(&input_container);
LOGE("=> Error encoding frames");
return ret;
}
if(got_frame) {
if (output_stream->codec->coded_frame->key_frame) {
encoded_packet.flags |= AV_PKT_FLAG_KEY;
}
encoded_packet.stream_index = output_stream->index;
encoded_packet.pts = av_rescale_q(current_frame_num, output_stream->codec->time_base, output_stream->time_base);
encoded_packet.dts = av_rescale_q(current_frame_num, output_stream->codec->time_base, output_stream->time_base);
ret = av_interleaved_write_frame(output_container, &encoded_packet);
if (ret < 0) {
cleanup(decoded_packet, output_container, decoded_frame);
avformat_close_input(&input_container);
LOGE("=> Error encoding frames");
return ret;
}
else {
current_frame_num +=1;
}
}
av_free_packet(&encoded_packet);
}
}
// audio packets
else if(decoded_packet.stream_index == audio_stream_index) {
// this is the trim range we're looking for
if(decoded_packet.pts >= starttime_int64_audio && decoded_packet.pts <= endtime_int64_audio) {
av_init_packet(&encoded_packet);
encoded_packet.data = decoded_packet.data;
encoded_packet.size = decoded_packet.size;
encoded_packet.stream_index = audio_stream_index;
encoded_packet.pts = av_rescale_q(current_frame_num_audio, output_stream_audio->codec->time_base, output_stream_audio->time_base);
encoded_packet.dts = av_rescale_q(current_frame_num_audio, output_stream_audio->codec->time_base, output_stream_audio->time_base);
ret = av_interleaved_write_frame(output_container, &encoded_packet);
if (ret < 0) {
cleanup(decoded_packet, output_container, decoded_frame);
avformat_close_input(&input_container);
LOGE("=> Error encoding frames");
return ret;
}
else {
current_frame_num_audio +=1;
}
av_free_packet(&encoded_packet);
}
}
}
Edit
I have slight improvement on the initial code. The audio and video are still not perfectly synced, but the original problem of the audio playing first followed by the video is resolved.
I'm now writing the decoded packet to the output container rather than re-encoding it.
In the end though I have the same problem - the trimmed video's audio and video streams are not perfectly synced.
// audio packets
else if(decoded_packet.stream_index == audio_stream_index) {
// this is the trim range we're looking for
if(decoded_packet.pts >= starttime_int64_audio && decoded_packet.pts <= endtime_int64_audio) {
ret = av_interleaved_write_frame(output_container, &decoded_packet);
if (ret < 0) {
cleanup(decoded_packet, output_container, decoded_frame);
avformat_close_input(&input_container);
LOGE("=> Error writing audio frame (%s)", av_err2str(ret));
return ret;
}
else {
current_frame_num_audio +=1;
}
}
else if(decoded_frame->pkt_pts > endtime_int64_audio) {
audio_copy_complete = true;
}
}
I believe you should be able to make this work if you use correctly set up both codec context and stream time bases, and then after encode_video2 and encode_audio2 you call av_packet_rescale_tb with such timebases.

Cropping Square Video using FFmpeg

Updated
So I am trying to decode a mp4 file, crop the video into a square, and then re encode it back out to another mp4 file. This is my current code but there are a few issues with it.
One is that the video doesn't keep its rotation after the video has been re encoded
Second is that the frames get outputted in a very fast video file that is not the same length as the original
Third is that there is no sound
Lastly and most importantly is do I need AVFilter to do the frame cropping or can it just be done per frame as a resize of the frame and then encoded back out.
const char *inputPath = "test.mp4";
const char *outPath = "cropped.mp4";
const char *outFileType = "mp4";
static AVFrame *oframe = NULL;
static AVFilterGraph *filterGraph = NULL;
static AVFilterContext *crop_ctx = NULL;
static AVFilterContext *buffersink_ctx = NULL;
static AVFilterContext *buffer_ctx = NULL;
int err;
int crop_video(int width, int height) {
av_register_all();
avcodec_register_all();
avfilter_register_all();
AVFormatContext *inCtx = NULL;
// open input file
err = avformat_open_input(&inCtx, inputPath, NULL, NULL);
if (err < 0) {
printf("error at open input in\n");
return err;
}
// get input file stream info
err = avformat_find_stream_info(inCtx, NULL);
if (err < 0) {
printf("error at find stream info\n");
return err;
}
// get info about video
av_dump_format(inCtx, 0, inputPath, 0);
// find video input stream
int vs = -1;
int s;
for (s = 0; s < inCtx->nb_streams; ++s) {
if (inCtx->streams[s] && inCtx->streams[s]->codec && inCtx->streams[s]->codec->codec_type == AVMEDIA_TYPE_VIDEO) {
vs = s;
break;
}
}
// check if video stream is valid
if (vs == -1) {
printf("error at open video stream\n");
return -1;
}
// set output format
AVOutputFormat * outFmt = av_guess_format(outFileType, NULL, NULL);
if (!outFmt) {
printf("error at output format\n");
return -1;
}
// get an output context to write to
AVFormatContext *outCtx = NULL;
err = avformat_alloc_output_context2(&outCtx, outFmt, NULL, NULL);
if (err < 0 || !outCtx) {
printf("error at output context\n");
return err;
}
// input and output stream
AVStream *outStrm = avformat_new_stream(outCtx, NULL);
AVStream *inStrm = inCtx->streams[vs];
// add a new codec for the output stream
AVCodec *codec = NULL;
avcodec_get_context_defaults3(outStrm->codec, codec);
outStrm->codec->thread_count = 1;
outStrm->codec->coder_type = AVMEDIA_TYPE_VIDEO;
if(outCtx->oformat->flags & AVFMT_GLOBALHEADER) {
outStrm->codec->flags |= CODEC_FLAG_GLOBAL_HEADER;
}
outStrm->codec->sample_aspect_ratio = outStrm->sample_aspect_ratio = inStrm->sample_aspect_ratio;
err = avio_open(&outCtx->pb, outPath, AVIO_FLAG_WRITE);
if (err < 0) {
printf("error at opening outpath\n");
return err;
}
outStrm->disposition = inStrm->disposition;
outStrm->codec->bits_per_raw_sample = inStrm->codec->bits_per_raw_sample;
outStrm->codec->chroma_sample_location = inStrm->codec->chroma_sample_location;
outStrm->codec->codec_id = inStrm->codec->codec_id;
outStrm->codec->codec_type = inStrm->codec->codec_type;
if (!outStrm->codec->codec_tag) {
if (! outCtx->oformat->codec_tag
|| av_codec_get_id (outCtx->oformat->codec_tag, inStrm->codec->codec_tag) == outStrm->codec->codec_id
|| av_codec_get_tag(outCtx->oformat->codec_tag, inStrm->codec->codec_id) <= 0) {
outStrm->codec->codec_tag = inStrm->codec->codec_tag;
}
}
outStrm->codec->bit_rate = inStrm->codec->bit_rate;
outStrm->codec->rc_max_rate = inStrm->codec->rc_max_rate;
outStrm->codec->rc_buffer_size = inStrm->codec->rc_buffer_size;
const size_t extra_size_alloc = (inStrm->codec->extradata_size > 0) ?
(inStrm->codec->extradata_size + FF_INPUT_BUFFER_PADDING_SIZE) :
0;
if (extra_size_alloc) {
outStrm->codec->extradata = (uint8_t*)av_mallocz(extra_size_alloc);
memcpy( outStrm->codec->extradata, inStrm->codec->extradata, inStrm->codec->extradata_size);
}
outStrm->codec->extradata_size = inStrm->codec->extradata_size;
AVRational input_time_base = inStrm->time_base;
AVRational frameRate = {25, 1};
if (inStrm->r_frame_rate.num && inStrm->r_frame_rate.den
&& (1.0 * inStrm->r_frame_rate.num / inStrm->r_frame_rate.den < 1000.0)) {
frameRate.num = inStrm->r_frame_rate.num;
frameRate.den = inStrm->r_frame_rate.den;
}
outStrm->r_frame_rate = frameRate;
outStrm->codec->time_base = inStrm->codec->time_base;
outStrm->codec->pix_fmt = inStrm->codec->pix_fmt;
outStrm->codec->width = width;
outStrm->codec->height = height;
outStrm->codec->has_b_frames = inStrm->codec->has_b_frames;
if (!outStrm->codec->sample_aspect_ratio.num) {
AVRational r0 = {0, 1};
outStrm->codec->sample_aspect_ratio =
outStrm->sample_aspect_ratio =
inStrm->sample_aspect_ratio.num ? inStrm->sample_aspect_ratio :
inStrm->codec->sample_aspect_ratio.num ?
inStrm->codec->sample_aspect_ratio : r0;
}
avformat_write_header(outCtx, NULL);
filterGraph = avfilter_graph_alloc();
if (!filterGraph) {
printf("could not open filter graph");
return -1;
}
AVFilter *crop = avfilter_get_by_name("crop");
AVFilter *buffer = avfilter_get_by_name("buffer");
AVFilter *buffersink = avfilter_get_by_name("buffersink");
char args[512];
snprintf(args, sizeof(args), "video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d",
width, height, inStrm->codec->pix_fmt,
inStrm->codec->time_base.num, inStrm->codec->time_base.den,
inStrm->codec->sample_aspect_ratio.num, inStrm->codec->sample_aspect_ratio.den);
err = avfilter_graph_create_filter(&buffer_ctx, buffer, NULL, args, NULL, filterGraph);
if (err < 0) {
printf("error initializing buffer filter\n");
return err;
}
err = avfilter_graph_create_filter(&buffersink_ctx, buffersink, NULL, NULL, NULL, filterGraph);
if (err < 0) {
printf("unable to create buffersink filter\n");
return err;
}
snprintf(args, sizeof(args), "%d:%d", width, height);
err = avfilter_graph_create_filter(&crop_ctx, crop, NULL, args, NULL, filterGraph);
if (err < 0) {
printf("error initializing crop filter\n");
return err;
}
err = avfilter_link(buffer_ctx, 0, crop_ctx, 0);
if (err < 0) {
printf("error linking filters\n");
return err;
}
err = avfilter_link(crop_ctx, 0, buffersink_ctx, 0);
if (err < 0) {
printf("error linking filters\n");
return err;
}
err = avfilter_graph_config(filterGraph, NULL);
if (err < 0) {
printf("error configuring the filter graph\n");
return err;
}
printf("filtergraph configured\n");
for (;;) {
AVPacket packet = {0};
av_init_packet(&packet);
err = AVERROR(EAGAIN);
while (AVERROR(EAGAIN) == err)
err = av_read_frame(inCtx, &packet);
if (err < 0) {
if (AVERROR_EOF != err && AVERROR(EIO) != err) {
printf("eof error\n");
return 1;
} else {
break;
}
}
if (packet.stream_index == vs) {
//
// AVPacket pkt_temp_;
// memset(&pkt_temp_, 0, sizeof(pkt_temp_));
// AVPacket *pkt_temp = &pkt_temp_;
//
// *pkt_temp = packet;
//
// int error, got_frame;
// int new_packet = 1;
//
// error = avcodec_decode_video2(inStrm->codec, frame, &got_frame, pkt_temp);
// if(error < 0) {
// LOGE("error %d", error);
// }
//
// // if (error >= 0) {
//
// // push the video data from decoded frame into the filtergraph
// int err = av_buffersrc_write_frame(buffer_ctx, frame);
// if (err < 0) {
// LOGE("error writing frame to buffersrc");
// return -1;
// }
// // pull filtered video from the filtergraph
// for (;;) {
// int err = av_buffersink_get_frame(buffersink_ctx, oframe);
// if (err == AVERROR_EOF || err == AVERROR(EAGAIN))
// break;
// if (err < 0) {
// LOGE("error reading buffer from buffersink");
// return -1;
// }
// }
//
// LOGI("output frame");
err = av_interleaved_write_frame(outCtx, &packet);
if (err < 0) {
printf("error at write frame");
return -1;
}
//}
}
av_free_packet(&packet);
}
av_write_trailer(outCtx);
if (!(outCtx->oformat->flags & AVFMT_NOFILE) && outCtx->pb)
avio_close(outCtx->pb);
avformat_free_context(outCtx);
avformat_close_input(&inCtx);
return 0;
}
It looks like you are looking for vf_crop.
The CropContext contains a x, y, w, and h, which should be what you need to crop a video to a specific width and height.

How to cut video with FFmpeg C API

How can I cut video with FFmpeg C API? From 00:10:00 to 00:20:00 for example.
What functions I need to use?
I convert video using this function:
int convert(char *file) {
AVFrame *frame;
AVPacket inPacket, outPacket;
if(avio_open(&outFormatContext->pb, file, AVIO_FLAG_WRITE) < 0) {
fprintf(stderr, "convert(): cannot open out file\n");
return 0;
}
avformat_write_header(outFormatContext, NULL);
frame = avcodec_alloc_frame();
av_init_packet(&inPacket);
while(av_read_frame(inFormatContext, &inPacket) >= 0) {
if(inPacket.stream_index == inVideoStreamIndex) {
avcodec_decode_video2(inCodecContext, frame, &frameFinished, &inPacket);
if(frameFinished) {
av_init_packet(&outPacket);
avcodec_encode_video2(outCodecContext, &outPacket, frame, &outputed);
if(outputed) {
if (av_write_frame(outFormatContext, &outPacket) != 0) {
fprintf(stderr, "convert(): error while writing video frame\n");
return 0;
}
}
av_free_packet(&outPacket);
}
}
}
av_write_trailer(outFormatContext);
av_free_packet(&inPacket);
return 1;
}
As Wagner Patriota says, "if you just wanna cut the video, you don't need to reencode the video if you want". Here is the code based on ffmpeg remuxing.c example that you don't need to reencode the video.
#include <libavutil/timestamp.h>
#include <libavformat/avformat.h>
static void log_packet(const AVFormatContext *fmt_ctx, const AVPacket *pkt, const char *tag)
{
AVRational *time_base = &fmt_ctx->streams[pkt->stream_index]->time_base;
printf("%s: pts:%s pts_time:%s dts:%s dts_time:%s duration:%s duration_time:%s stream_index:%d\n",
tag,
av_ts2str(pkt->pts), av_ts2timestr(pkt->pts, time_base),
av_ts2str(pkt->dts), av_ts2timestr(pkt->dts, time_base),
av_ts2str(pkt->duration), av_ts2timestr(pkt->duration, time_base),
pkt->stream_index);
}
int cut_video(double from_seconds, double end_seconds, const char* in_filename, const char* out_filename) {
AVOutputFormat *ofmt = NULL;
AVFormatContext *ifmt_ctx = NULL, *ofmt_ctx = NULL;
AVPacket pkt;
int ret, i;
av_register_all();
if ((ret = avformat_open_input(&ifmt_ctx, in_filename, 0, 0)) < 0) {
fprintf(stderr, "Could not open input file '%s'", in_filename);
goto end;
}
if ((ret = avformat_find_stream_info(ifmt_ctx, 0)) < 0) {
fprintf(stderr, "Failed to retrieve input stream information");
goto end;
}
av_dump_format(ifmt_ctx, 0, in_filename, 0);
avformat_alloc_output_context2(&ofmt_ctx, NULL, NULL, out_filename);
if (!ofmt_ctx) {
fprintf(stderr, "Could not create output context\n");
ret = AVERROR_UNKNOWN;
goto end;
}
ofmt = ofmt_ctx->oformat;
for (i = 0; i < ifmt_ctx->nb_streams; i++) {
AVStream *in_stream = ifmt_ctx->streams[i];
AVStream *out_stream = avformat_new_stream(ofmt_ctx, in_stream->codec->codec);
if (!out_stream) {
fprintf(stderr, "Failed allocating output stream\n");
ret = AVERROR_UNKNOWN;
goto end;
}
ret = avcodec_copy_context(out_stream->codec, in_stream->codec);
if (ret < 0) {
fprintf(stderr, "Failed to copy context from input to output stream codec context\n");
goto end;
}
out_stream->codec->codec_tag = 0;
if (ofmt_ctx->oformat->flags & AVFMT_GLOBALHEADER)
out_stream->codec->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
}
av_dump_format(ofmt_ctx, 0, out_filename, 1);
if (!(ofmt->flags & AVFMT_NOFILE)) {
ret = avio_open(&ofmt_ctx->pb, out_filename, AVIO_FLAG_WRITE);
if (ret < 0) {
fprintf(stderr, "Could not open output file '%s'", out_filename);
goto end;
}
}
ret = avformat_write_header(ofmt_ctx, NULL);
if (ret < 0) {
fprintf(stderr, "Error occurred when opening output file\n");
goto end;
}
// int indexs[8] = {0};
// int64_t start_from = 8*AV_TIME_BASE;
ret = av_seek_frame(ifmt_ctx, -1, from_seconds*AV_TIME_BASE, AVSEEK_FLAG_ANY);
if (ret < 0) {
fprintf(stderr, "Error seek\n");
goto end;
}
int64_t *dts_start_from = malloc(sizeof(int64_t) * ifmt_ctx->nb_streams);
memset(dts_start_from, 0, sizeof(int64_t) * ifmt_ctx->nb_streams);
int64_t *pts_start_from = malloc(sizeof(int64_t) * ifmt_ctx->nb_streams);
memset(pts_start_from, 0, sizeof(int64_t) * ifmt_ctx->nb_streams);
while (1) {
AVStream *in_stream, *out_stream;
ret = av_read_frame(ifmt_ctx, &pkt);
if (ret < 0)
break;
in_stream = ifmt_ctx->streams[pkt.stream_index];
out_stream = ofmt_ctx->streams[pkt.stream_index];
log_packet(ifmt_ctx, &pkt, "in");
if (av_q2d(in_stream->time_base) * pkt.pts > end_seconds) {
av_free_packet(&pkt);
break;
}
if (dts_start_from[pkt.stream_index] == 0) {
dts_start_from[pkt.stream_index] = pkt.dts;
printf("dts_start_from: %s\n", av_ts2str(dts_start_from[pkt.stream_index]));
}
if (pts_start_from[pkt.stream_index] == 0) {
pts_start_from[pkt.stream_index] = pkt.pts;
printf("pts_start_from: %s\n", av_ts2str(pts_start_from[pkt.stream_index]));
}
/* copy packet */
pkt.pts = av_rescale_q_rnd(pkt.pts - pts_start_from[pkt.stream_index], in_stream->time_base, out_stream->time_base, AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX);
pkt.dts = av_rescale_q_rnd(pkt.dts - dts_start_from[pkt.stream_index], in_stream->time_base, out_stream->time_base, AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX);
if (pkt.pts < 0) {
pkt.pts = 0;
}
if (pkt.dts < 0) {
pkt.dts = 0;
}
pkt.duration = (int)av_rescale_q((int64_t)pkt.duration, in_stream->time_base, out_stream->time_base);
pkt.pos = -1;
log_packet(ofmt_ctx, &pkt, "out");
printf("\n");
ret = av_interleaved_write_frame(ofmt_ctx, &pkt);
if (ret < 0) {
fprintf(stderr, "Error muxing packet\n");
break;
}
av_free_packet(&pkt);
}
free(dts_start_from);
free(pts_start_from);
av_write_trailer(ofmt_ctx);
end:
avformat_close_input(&ifmt_ctx);
/* close output */
if (ofmt_ctx && !(ofmt->flags & AVFMT_NOFILE))
avio_closep(&ofmt_ctx->pb);
avformat_free_context(ofmt_ctx);
if (ret < 0 && ret != AVERROR_EOF) {
fprintf(stderr, "Error occurred: %s\n", av_err2str(ret));
return 1;
}
return 0;
}
If you just wanna cut the video, you don't need to reencode the video if you want.
So I am supposing you wanna cut and reencode for some reason. So, based on your code:
Observe you must to have access to the video AVStream* structure... I named it as inVideoStream.
int convert_and_cut(char *file, float starttime, float endtime) {
AVFrame *frame;
AVPacket inPacket, outPacket;
if(avio_open(&outFormatContext->pb, file, AVIO_FLAG_WRITE) < 0) {
fprintf(stderr, "convert(): cannot open out file\n");
return 0;
}
// seek to the start time you wish.
// BEGIN
AVRational default_timebase;
default_timebase.num = 1;
default_timebase.den = AV_TIME_BASE;
// suppose you have access to the "inVideoStream" of course
int64_t starttime_int64 = av_rescale_q((int64_t)( starttime * AV_TIME_BASE ), default_timebase, inVideoStream->time_base);
int64_t endtime_int64 = av_rescale_q((int64_t)( endtime * AV_TIME_BASE ), default_timebase, inVideoStream->time_base);
if(avformat_seek_file(inFormatContext, inVideoStreamIndex, INT64_MIN, starttime_int64, INT64_MAX, 0) < 0) {
// error... do something...
return 0; // usually 0 is used for success in C, but I am following your code.
}
avcodec_flush_buffers( inVideoStream->codec );
// END
avformat_write_header(outFormatContext, NULL);
frame = avcodec_alloc_frame();
av_init_packet(&inPacket);
// you used avformat_seek_file() to seek CLOSE to the point you want... in order to give precision to your seek,
// just go on reading the packets and checking the packets PTS (presentation timestamp)
while(av_read_frame(inFormatContext, &inPacket) >= 0) {
if(inPacket.stream_index == inVideoStreamIndex) {
avcodec_decode_video2(inCodecContext, frame, &frameFinished, &inPacket);
// this line guarantees you are getting what you really want.
if(frameFinished && frame->pkt_pts >= starttime_int64 && frame->pkt_pts <= endtime_int64) {
av_init_packet(&outPacket);
avcodec_encode_video2(outCodecContext, &outPacket, frame, &outputed);
if(outputed) {
if (av_write_frame(outFormatContext, &outPacket) != 0) {
fprintf(stderr, "convert(): error while writing video frame\n");
return 0;
}
}
av_free_packet(&outPacket);
}
// exit the loop if you got the frames you want.
if(frame->pkt_pts > endtime_int64) {
break;
}
}
}
av_write_trailer(outFormatContext);
av_free_packet(&inPacket);
return 1;
}
In addition, as ustin says, the code based on ffmpeg remuxing.c example.
add out_stream->time_base = in_stream->time_base; before out_stream->codec->codec_tag = 0;
add int64_t *start_from = NULL; after int stream_mapping_size = 0;
add start_from = av_mallocz_array(stream_mapping_size, sizeof(*start_from)); after stream_mapping = av_mallocz_array(stream_mapping_size, sizeof(*stream_mapping));
add start_from[stream_mapping[i]] = INT16_MIN; after stream_mapping[i] = stream_index++;
add
if (start_from[pkt.stream_index] == INT16_MIN) {
int64_t dts = pkt.dts;
int64_t pts = pkt.pts;
int64_t min_ts = dts > pts ? pts : dts;
start_from[pkt.stream_index] = min_ts < 0 ? 0 : min_ts;
}
After
if (pkt.stream_index >= stream_mapping_size ||
stream_mapping[pkt.stream_index] < 0) {
av_packet_unref(&pkt);
continue;
}
change justin code, about pts/dst set value to this:
pkt.pts = av_rescale_q_rnd(pkt.pts - start_from[pkt.stream_index], in_stream->time_base, out_stream->time_base, AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX);
pkt.dts = av_rescale_q_rnd(pkt.dts - start_from[pkt.stream_index], in_stream->time_base, out_stream->time_base, AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX);
add free(start_from); before end:
Tips: start/end timestamp like justin says

Resources