/* $Id: fors_img_screen_flat_impl.c,v 1.48 2013-09-10 19:16:19 cgarcia Exp $
 *
 * This file is part of the FORS Library
 * Copyright (C) 2002-2010 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 */

/*
 * $Author: cgarcia $
 * $Date: 2013-09-10 19:16:19 $
 * $Revision: 1.48 $
 * $Name: not supported by cvs2svn $
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <fors_img_screen_flat_impl.h>

#include <fors_stack.h>
#include <fors_tools.h>
#include <fors_dfs.h>
#include <fors_qc.h>
#include <fors_utils.h>
#include "fiera_config.h"
#include "fors_saturation.h"
#include "fors_detmodel.h"
#include "fors_overscan.h"
#include "fors_ccd_config.h"
#include "fors_subtract_bias.h"

#include <cpl.h>
#include <math.h>

/**
 * @addtogroup fors_img_screen_flat
 */

/**@{*/

const char *const fors_img_screen_flat_name = "fors_img_screen_flat";
const char *const fors_img_screen_flat_description_short = 
"Compute master screen flat frame";
const char *const fors_img_screen_flat_author = "Jonas M. Larsen";
const char *const fors_img_screen_flat_email = PACKAGE_BUGREPORT;
const char *const fors_img_screen_flat_description = 
"After bias subtraction, the input flat field frames are combined using\n"
"the given stack method. The combined frame is finally normalised dividing\n"
"it by its large scale illumination trend. The large scale trend is obtained\n"
"by applying a median filter with a large kernel. To avoid boundary effects, \n"
"the median filter is applied only to the specified region.\n"
"The overscan regions, if present, are removed from the result.\n\n"
"Input files:\n"
"\n"
"  DO category:               Type:       Explanation:         Required:\n"
"  SCREEN_FLAT_IMG            Raw         Screen flat field    Y\n"
"  MASTER_BIAS                Raw         Master bias          Y\n"
"\n"
"Output files:\n\n"
"  DO category:               Data type:  Explanation:\n"
"  MASTER_SCREEN_FLAT_IMG     FITS image  Master screen flat field\n";


static void
remove_large_scale(fors_image *master_screen_flat,
                   const cpl_parameterlist *parameters,
                   const char *context,
                   const fors_setting *setting);

static void
remove_large_scale_fit(fors_image *master_screen_flat,
                       const cpl_parameterlist *parameters,
                       const char *context,
                       const fors_setting *setting);

static void
write_qc(cpl_propertylist *qc, 
         const cpl_frame *first_raw,
         const fors_image_list *sflats, 
         const fors_image *master_sflat,
         const fors_setting *setting,
         double saturation);
/**
 * @brief    Define recipe parameters
 * @param    parameters     parameter list to fill
 */

void fors_img_screen_flat_define_parameters(cpl_parameterlist *parameters)
{
    char *context = cpl_sprintf("fors.%s", fors_img_screen_flat_name);
    char *full_name = NULL;
    
    fors_stack_define_parameters(parameters, context, "average");

    {
        cpl_parameter *p;
        const char *name;
        
        /* Median filter, xradius, yradius */
        name = "xradius";
        full_name = cpl_sprintf("%s.%s", context, name);
        p = cpl_parameter_new_value(full_name,
                                    CPL_TYPE_INT,
                                    "Median filter x radius (unbinned pixels)",
                                    context,
                                    50);
        cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, name);
        cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
        cpl_parameterlist_append(parameters, p);
        cpl_free((void *)full_name); full_name = NULL;
        
        name = "yradius";
        full_name = cpl_sprintf("%s.%s", context, name);
        p = cpl_parameter_new_value(full_name,
                                    CPL_TYPE_INT,
                                    "Median filter y radius (unbinned pixels)",
                                    context,
                                    50);
        cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, name);
        cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
        cpl_parameterlist_append(parameters, p);
        cpl_free((void *)full_name); full_name = NULL;

        name = "degree";
        full_name = cpl_sprintf("%s.%s", context, name);
        p = cpl_parameter_new_value(full_name,
                                    CPL_TYPE_INT,
                                    "Degree of fitting polynomial",
                                    context,
                                    -1);
        cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, name);
        cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
        cpl_parameterlist_append(parameters, p);
        cpl_free((void *)full_name); full_name = NULL;

        name = "sampling";
        full_name = cpl_sprintf("%s.%s", context, name);
        p = cpl_parameter_new_value(full_name,
                                    CPL_TYPE_INT,
                                    "Sampling interval for fitting",
                                    context,
                                    100);
        cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, name);
        cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
        cpl_parameterlist_append(parameters, p);
        cpl_free((void *)full_name); full_name = NULL;
    }

    cpl_free((void *)context);
    cpl_free((void *)full_name);

    return;
}

#undef cleanup
#define cleanup \
do { \
    cpl_frameset_delete(sflat_frames); \
    fors_setting_delete(&setting); \
    cpl_frameset_delete(master_bias_frame); \
    fors_image_delete(&master_bias); \
    fors_stack_method_delete(&sm); \
    cpl_free((void *)context); \
    fors_image_delete(&master_screen_flat); \
    fors_image_delete(&master_norm_flat); \
    fors_image_list_delete(&sflats, fors_image_delete); \
    cpl_propertylist_delete(qc); \
} while (0)
/**
 * @brief    Do the processing
 *
 * @param    frames         input frames
 * @param    parameters     The parameters list
 *
 * @return   0 if everything is ok
 */
void
fors_img_screen_flat(cpl_frameset *frames, const cpl_parameterlist *parameters)
{
    /* Raw */
    cpl_frameset *sflat_frames      = NULL;
    fors_image_list *sflats   = NULL;
    fors_setting *setting           = NULL;

    /* Calibration */
    cpl_frameset *master_bias_frame = NULL;
    fors_image *master_bias   = NULL; 

    /* Products */
    fors_image *master_screen_flat = NULL;
    fors_image *master_norm_flat   = NULL;
    
    /* QC */
    cpl_propertylist *qc = cpl_propertylist_new();
    double saturation;

    /* Parameters */
    stack_method *sm = NULL;

    /* Other */
    char *context = cpl_sprintf("fors.%s", fors_img_screen_flat_name);

    /* Get parameters */
    sm = fors_stack_method_new(parameters, context);
    assure( !cpl_error_get_code(), return, "Could not get stacking method");
    
    /* Find raw */
    sflat_frames = fors_frameset_extract(frames, SCREEN_FLAT_IMG);
    assure( cpl_frameset_get_size(sflat_frames) > 0, return, 
            "No %s provided", SCREEN_FLAT_IMG);
    
    /* Find calibration */
    master_bias_frame = fors_frameset_extract(frames, MASTER_BIAS);
    assure( cpl_frameset_get_size(master_bias_frame) == 1, return, 
            "One %s required. %"CPL_SIZE_FORMAT" found", 
            MASTER_BIAS, cpl_frameset_get_size(master_bias_frame));
    
    /* Get instrument setting */
    setting = fors_setting_new(cpl_frameset_get_position(sflat_frames, 0));
    cpl_propertylist * sflat_header = 
            cpl_propertylist_load(cpl_frame_get_filename(
                    cpl_frameset_get_position(sflat_frames, 0)), 0);
    fors::fiera_config sflat_ccd_config(sflat_header);
    cpl_propertylist_delete(sflat_header);
    assure( !cpl_error_get_code(), return, "Could not get instrument setting" );

    /* Update RON estimation from bias */
    cpl_propertylist * master_bias_header =
       cpl_propertylist_load(cpl_frame_get_filename(
               cpl_frameset_get_position(master_bias_frame, 0)), 0);
    fors::update_ccd_ron(sflat_ccd_config, master_bias_header);
    cpl_propertylist_delete(master_bias_header);
    assure( !cpl_error_get_code(), return, "Could not get RON from master bias");

    master_bias =
            fors_image_load(cpl_frameset_get_position(master_bias_frame, 0));
    assure( !cpl_error_get_code(), return, 
            "Could not load master bias");

    /* Load raw frames*/
    fors_image_list * sflats_raw = 
        fors_image_load_list(sflat_frames);
    assure( !cpl_error_get_code(), return, "Could not load screen flat images");

    /* Create variances map */
    fors_image_variance_from_detmodel(sflats_raw, sflat_ccd_config);
    /* Subtract overscan */
    sflats = fors_subtract_prescan(sflats_raw, sflat_ccd_config);
    assure( !cpl_error_get_code(), return, "Could not subtract overscan");

    /* Trimm pre/overscan */
    fors_trimm_preoverscan(sflats, sflat_ccd_config);
    fors_image_list_delete(&sflats_raw, fors_image_delete);
    assure( !cpl_error_get_code(), return, "Could not trimm overscan");

    /* Subtract master bias */
    fors_subtract_bias_imglist(sflats, master_bias);
    saturation = fors_saturation_imglist_satper(sflats);
    assure( !cpl_error_get_code(), return, "Could not subtract master bias");

    /* Stack */
    master_screen_flat = fors_stack_const(sflats, sm);
    assure( !cpl_error_get_code(), return, "Screen flat stacking failed");

    /* Divide out large scale structure */
    master_norm_flat = fors_image_duplicate(master_screen_flat);
    remove_large_scale(master_norm_flat,
                       parameters, context,
                       setting);

    assure( !cpl_error_get_code(), return, 
            "Flat field smoothing/normalization failed");

    /* QC */
    write_qc(qc, cpl_frameset_get_position(sflat_frames, 0),
             sflats, master_norm_flat, setting, saturation);
    assure( !cpl_error_get_code(), return, "Failed to compute QC");

    /* Save products */
    fors_dfs_save_image_err(frames, master_screen_flat, MASTER_SCREEN_FLAT_IMG,
                        NULL, NULL, parameters, fors_img_screen_flat_name, 
                        cpl_frameset_get_position(sflat_frames, 0));
    assure( !cpl_error_get_code(), return, "Saving %s failed",
            MASTER_SCREEN_FLAT_IMG);
    
    fors_dfs_save_image_err(frames, master_norm_flat, MASTER_NORM_FLAT_IMG,
                        qc, NULL, parameters, fors_img_screen_flat_name, 
                        cpl_frameset_get_position(sflat_frames, 0));
    assure( !cpl_error_get_code(), return, "Saving %s failed",
            MASTER_NORM_FLAT_IMG);
    
    cleanup;
    return;
}

#undef cleanup
#define cleanup \
do { \
    cpl_image_delete(smoothed); \
} while (0)
/**
 * @brief    Fit and divide out large scale structure
 *
 * @param    master_screen_flat    pre-normalized screen flat image
 * @param    parameters            smoothing parameters
 * @param    context               read parameters from this context
 * @param    setting               instrument setting
 *
 * This function applies a median filter to the screen flat field image.
 * The master screen flat field is then divided by this smoothed image
 * which removes large scale structure and normalizes the flat field to one.
 *
 * To avoid dealing with correlated noise, the obtained smoothed image is
 * assumed to be effectively noiseless. For a median filter window size of
 * only 11x11, this assumption introduces a relative error in the error bars 
 * of only ~ 1/sqrt(11x11) = 10 %.
 *
 */
static void
remove_large_scale(fors_image *master_screen_flat,
                   const cpl_parameterlist *parameters,
                   const char *context,
                   const fors_setting *setting)

{
    cpl_image *smoothed = NULL;
    char *name;
    int xradius, yradius;
    int xstart, ystart;
    int xend, yend;

    name = cpl_sprintf("%s.%s", context, "degree");
    const cpl_parameter *param = cpl_parameterlist_find_const(parameters, name);
    int degree = cpl_parameter_get_int(param);

    cpl_free((void *)name); name = NULL;
    if (degree >= 0) {
        remove_large_scale_fit(master_screen_flat, parameters, context,
                               setting);
        return;
    }

    cpl_msg_info(cpl_func, "Median filter parameters:");
    
    cpl_msg_indent_more();
    cpl_msg_indent_more();
    name = cpl_sprintf("%s.%s", context, "xradius");
    xradius = dfs_get_parameter_int_const(parameters, 
                                          name);
    cpl_free((void *)name); name = NULL;
    cpl_msg_indent_less();
    cpl_msg_indent_less();
    assure( !cpl_error_get_code(), return, NULL );
    

    cpl_msg_indent_more();
    cpl_msg_indent_more();
    name = cpl_sprintf("%s.%s", context, "yradius");
    yradius = dfs_get_parameter_int_const(parameters, 
                                          name);
    cpl_free((void *)name); name = NULL;
    cpl_msg_indent_less();
    cpl_msg_indent_less();
    assure( !cpl_error_get_code(), return, NULL );
    
    /* Done reading parameters, apply filter */
    bool use_data = true;  /* Filter the data values, not error bars */

    /* Correct for CCD binning */
    xradius = (xradius - 1)/setting->binx + 1;
    yradius = (yradius - 1)/setting->biny + 1;
    xstart = 1;
    ystart = 1;
    xend = fors_image_get_size_x(master_screen_flat);
    yend = fors_image_get_size_y(master_screen_flat);
    int xstep = 1;
    int ystep = 1;

    smoothed = fors_image_filter_median_create(master_screen_flat,
                                               xradius, yradius,
                                               xstart, ystart,
                                               xend, yend,
                                               xstep, ystep,
                                               use_data);
    
    assure( !cpl_error_get_code(), return, NULL );

#ifdef DEBUG
    {
        const char *filename = "smooth.fits";

        cpl_msg_warning(cpl_func, "Saving large scale structure image to %s",
                     filename);
        cpl_image_save(smoothed, filename, CPL_BPP_IEEE_FLOAT,
                       NULL, CPL_IO_DEFAULT);
    }
#endif
    
    fors_image_divide_noerr(master_screen_flat, smoothed);
    /* smoothed image modified here */

    cleanup;
    return;
}



#undef cleanup
#define cleanup \
do { \
    cpl_image_delete(smoothed); \
} while (0)
/**
 * @brief    Fit and divide out large scale structure
 *
 * @param    master_screen_flat    pre-normalized screen flat image
 * @param    parameters            fitting parameters
 * @param    context               read parameters from this context
 * @param    setting               instrument setting
 *
 * This function fits the illuminated part of the screen flat field frame
 * with a low degree polynomial. The input frame is then divided by this 
 * fit-smoothed image which removes large scale structure and normalizes 
 * the flat field to one.
 *
 * To avoid dealing with correlated noise, the obtained smoothed image is
 * assumed to be effectively noiseless.
 */
static void
remove_large_scale_fit(fors_image *master_screen_flat,
                       const cpl_parameterlist *parameters,
                       const char *context,
                       const fors_setting *setting)

{
    cpl_image *smoothed = NULL;
    char *name;
    int step;
    int degree;
    (void)setting;

    cpl_msg_info(cpl_func, "Large scale fitting parameters:");
    
    cpl_msg_indent_more();
    cpl_msg_indent_more();
    name = cpl_sprintf("%s.%s", context, "sampling");
    step = dfs_get_parameter_int_const(parameters, 
                                       name);
    cpl_free((void *)name); name = NULL;
    cpl_msg_indent_less();
    cpl_msg_indent_less();
    assure( !cpl_error_get_code(), return, NULL );

    cpl_msg_indent_more();
    cpl_msg_indent_more();
    name = cpl_sprintf("%s.%s", context, "degree");
    degree = dfs_get_parameter_int_const(parameters,
                                         name);
    cpl_free((void *)name); name = NULL;
    cpl_msg_indent_less();
    cpl_msg_indent_less();
    assure( !cpl_error_get_code(), return, NULL );
    
    /*
     * Determine typical level of flat field, that will be used to
     * avoid fitting values from the vignetted region.
     */

    float level = fors_image_get_median(master_screen_flat, NULL) / 2;

    smoothed = fors_image_flat_fit_create(master_screen_flat,
                                          step, degree, level);
    
    assure( !cpl_error_get_code(), return, NULL );

#ifdef DEBUG
    {
        const char *filename = "smooth.fits";

        cpl_msg_info(cpl_func, "Saving large scale structure image to %s",
                     filename);
        cpl_image_save(smoothed, filename, CPL_BPP_IEEE_FLOAT,
                       NULL, CPL_IO_DEFAULT);
    }
#endif
    
    fors_image_divide_noerr(master_screen_flat, smoothed);
    /* smoothed image modified here */

    cleanup;
    return;
}



#undef cleanup
#define cleanup \
do { \
    fors_image_delete(&image); \
} while (0)
/**
 * @brief    Compute QC
 *
 * @param    qc                write to this list
 * @param    sflats            raw flats (already bias subtracted)
 * @param    setting           instrument setting
 * @param    saturation        previously computed saturation percentage
 */
static void
write_qc(cpl_propertylist *qc, 
         const cpl_frame *first_raw_frame,
         const fors_image_list *sflats, 
         const fors_image *master_sflat,
         const fors_setting *setting,
         double saturation)
{
    fors_image *image = NULL;

    fors_qc_start_group(qc, fors_qc_dic_version, setting->instrument);

    fors_qc_write_group_heading(first_raw_frame,
                                MASTER_SCREEN_FLAT_IMG,
                                setting->instrument);
    assure( !cpl_error_get_code(), return, "Could not write %s QC parameters", 
            MASTER_SCREEN_FLAT_IMG);



    fors_qc_write_qc_double(qc,
                            saturation,
                            "QC.OVEREXPO",
                            "%",
                            "Percentage of overexposed pixels",
                            setting->instrument);

    /* 
     * Second_raw is NULL if only one raw frame provided:
     */

    const fors_image *first_raw = fors_image_list_first_const(sflats);
    const fors_image *second_raw = fors_image_list_next_const(sflats); 
        
    fors_qc_write_qc_double(qc,
                            /* Median of bias subtracted frame */
                            fors_image_get_median(first_raw, NULL)
                            / setting->exposure_time,
                            "QC.FLAT.EFF",
                            "ADU/s",
                            "Flat field lamp efficiency",
                            setting->instrument);


    /*
     * Here compute QC.FLAT.PHN and QC.FLAT.CONAD:
     */

    double master_photon_noise;
    double conad, conad_err;

    /* FIXME: 
     * Here in principle we should use the propagated errors to get
     * the master flat photon noise, instead of computing it all over
     * again...
     */

    if (second_raw != NULL) {
            
        /* 
         * Use central 101x101 window of difference frame 
         */

        if (fors_image_get_size_x(second_raw) >= 101 &&
            fors_image_get_size_y(second_raw) >= 101) {

            fors_image *center_first = fors_image_duplicate(first_raw);
            fors_image *center_second = fors_image_duplicate(second_raw);
                
            int mid_x = (fors_image_get_size_x(center_first) + 1) / 2;
            int mid_y = (fors_image_get_size_y(center_first) + 1) / 2;

            fors_image_crop(center_first, 
                            mid_x - 50, mid_y - 50,
                            mid_x + 50, mid_y + 50);

            fors_image_crop(center_second, 
                            mid_x - 50, mid_y - 50,
                            mid_x + 50, mid_y + 50);

            /*
             * This one is the intensity image, will be used later
             * for gain computation
             */

            image = fors_image_duplicate(center_first);

            /*
             * 1) Determine the median value of the ratio  F1/F2 = k .
             * 2) Compute the frame  F1 - k*F2 (whose mean should be
             *    around 0).
             * 3) The expected variance of  F1 - k*F2  is  V = V1 + k*k*V2,
             *    where V1 and V2 are the variances of the frames F1 and F2.
             * 4) Neglecting the contribution of the readout noise, it is  
             *    V1 = F1/conad  and  V2 = F2/conad, so it is also V1/V2 = k .
             * 5) Therefore we get V = V1 + k*V1, that is   V1 = V/(1+k),
             *    an estimator of the variance of the first frame.
             * 6) The noise (1-sigma level) of the first frame is then
             *    sqrt(V1) = sqrt(V)/sqrt(1+k).
             */

            /* 1) */
            fors_image *ratio = fors_image_duplicate(center_first);
            fors_image_divide(ratio, center_second);
            double k = fors_image_get_median(ratio, NULL);
            fors_image_delete(&ratio);

            /* 2) */
            fors_image_multiply_scalar(center_second, k, -1);
            fors_image_subtract(center_first, center_second);
            fors_image_delete(&center_second);

            /* 6) */
            master_photon_noise = fors_image_get_stdev(center_first, NULL);
            master_photon_noise /= sqrt(1+k);

            /*
             * Now estimate the noise of all frames (assuming that all
             * frames have similar exposure times...)
             */

            master_photon_noise /= sqrt(fors_image_list_size(sflats));

            /*
             * Now center_first is used to determine the CONAD: the
             * center_first is still to be squared and divided by (1+k) 
             * to become an estimator of the variance of the first frame.
             *
             * CONAD is estimated as 1 / mean(variance_i / flux_i)
             *
             * Note: It is important that the average is made using
             * (variance_i / flux_i), and not using (flux_i / variance_i)
             * because the variance estimates are very noisy and
             * close to zero.
             */

            fors_image_square(center_first);
            fors_image_divide(center_first, image);

            /*
             * We _must_ use mean, here, because the mean of the
             * squares in center_first is an estimate of the variance,
             * but the median is not (asymmetric distribution).
             */

            double gain = fors_image_get_mean(center_first, NULL) / (1+k);
            double gain_err = fors_image_get_stdev(center_first, NULL) / (1+k);
            fors_image_delete(&center_first);
            fors_image_delete(&image);

            /*
             * Relative error in CONAD and GAIN is the same:
             */

            conad = 1.0 / gain;
            conad_err = conad * gain_err/gain;
            conad_err /= 101;
        }
        else {
            cpl_msg_warning(cpl_func,
                            "Raw images too small (%"CPL_SIZE_FORMAT
                            "x%"CPL_SIZE_FORMAT"), "
                            "need size 101x101 to compute master flat "
                            "photon noise and gain",
                            fors_image_get_size_x(second_raw),
                            fors_image_get_size_y(second_raw));

            master_photon_noise = -1;
        }
    }
    else {
        cpl_msg_warning(cpl_func, "Only 1 raw frame provided, "
                        "cannot compute master flat photon noise");
        master_photon_noise = -1;
    }

    fors_qc_write_qc_double(qc,
                            master_photon_noise,
                            "QC.FLAT.PHN",
                            "ADU",
                            "Photon noise in master screen flat field",
                            setting->instrument);
        
    double master_fixed_pattern_noise = 
        fors_fixed_pattern_noise(master_sflat,
                                 fors_image_get_median(first_raw, NULL),
                                 master_photon_noise);
    assure( !cpl_error_get_code(), return, 
            "Could not compute fixed pattern noise" );
    
    fors_qc_write_qc_double(qc,
                            master_fixed_pattern_noise,
                            "QC.FLAT.FPN",
                            "ADU",
                            "Fixed-pattern noise",
                            setting->instrument);

    master_fixed_pattern_noise /= fors_image_get_median(first_raw, NULL);

    fors_qc_write_qc_double(qc,
                            master_fixed_pattern_noise,
                            "QC.FLAT.FPN.REL",
                            NULL,
                            "Relative fixed-pattern noise",
                            setting->instrument);

    fors_qc_write_qc_double(qc,
                            conad,
                            "QC.FLAT.CONAD",
                            "e-/ADU",
                            "Conversion factor from ADU to electrons",
                            setting->instrument);

    fors_qc_write_qc_double(qc,
                            conad_err,
                            "QC.FLAT.CONADERR",
                            "e-/ADU",
                            "Error on conversion factor ADU/electrons",
                            setting->instrument);

#define SALTA 1
#ifndef SALTA
    /* 
     * Estimate CONAD as 1 / mean(variance_i / flux_i) 
     *
     * where variance_i is estimated as (flux_i1 - flux_i2)^2/2
     * for the fluxes of the first two raw frames. The systematic 
     * part of the difference (i.e. due to sligthly different 
     * exposure times) is subtracted from the difference image.
     * 
     * Note: It is important that the average is made using
     * (variance_i / flux_i), and not using (flux_i / variance_i)
     * because the variance estimates are very noisy and
     * close to zero.
     */
        
    double conad, conad_err;

    if (second_raw != NULL) {
        image = fors_image_duplicate(first_raw);
        fors_image_subtract(image, second_raw);

        /*
         * At every pixel, variance_i / flux_i is an estimate of the gain.
         *  
         * To get the average use only part of image with flux_i above 
         * 0.5*median, because regions with zero ~flux cause division by 
         * zero and do not help estimate the gain. 
         */

        double gain_mean, gain_err;

        {
            double first_raw_median = fors_image_get_median(first_raw, NULL);

            const float *first_raw_data = fors_image_get_data_const(first_raw);
            const float *diff_data = fors_image_get_data_const(image);

            const int Ntot = fors_image_get_size_x(image) 
                           * fors_image_get_size_y(image);

            /*
             * Computing the mean difference between the two raw flats,
             * and subtract it from the difference image. This is in 
             * principle incorrect: the difference between two flat
             * exposures is not a constant, but it has the same illumination
             * distribution as the original flats:
             * 
             *   diff = flat1 - flat2 = flat1 - k*flat1 = (1-k) * flat1
             *
             * Correcting for a constant offset would not prevent an
             * overestimation of the variance in case k is different 
             * from 1.0!
             */

            double diff_mean = 0;
            int N = 0;
            int i;
            for (i = 0; i < Ntot; i++) {
                if (first_raw_data[i] > 0.5 * first_raw_median) {
                    diff_mean += diff_data[i];
                    N += 1;
                }
            }
            diff_mean /= N;
            fors_image_subtract_scalar(image, diff_mean, -1);

            /* 
             * Pixel per pixel estimate of variance.
             */

            fors_image_square(image);
            fors_image_divide_scalar(image, 2, -1);

            /* 
             * Now image contains estimates of variance at every pixel,
             * i.e., diff_data[] points to variance values.
             */

            double sum_vf = 0;     /* sum (variance_i / flux_i)   */
            double sum_vfvf = 0;   /* sum (variance_i / flux_i)^2 */
            N = 0;
            for (i = 0; i < Ntot; i++) {
                if (first_raw_data[i] > 0.5 * first_raw_median) {
                    double vf = diff_data[i] / first_raw_data[i];
                    sum_vf   += vf;
                    sum_vfvf += vf*vf;
                    N += 1;
                }
            }

            gain_mean = sum_vf / N;
            double vfvf_mean = sum_vfvf / N;
            gain_err = sqrt(fabs(vfvf_mean - gain_mean*gain_mean));

            /* 
             * Iterate, rejecting 5 sigma outliers (e.g. due to cosmics
             * in any raw frame).
             */

            double hicut = gain_mean + 5*gain_err;
            sum_vf = 0;  
            sum_vfvf = 0;
            N = 0;
            for (i = 0; i < Ntot; i++) {
                if (first_raw_data[i] > 0.5 * first_raw_median) {
                    double vf = diff_data[i] / first_raw_data[i];
                    if (vf < hicut) {
                        sum_vf   += vf;
                        sum_vfvf += vf*vf;
                        N += 1;
                    }
                }
            }
            
            cpl_msg_info(cpl_func, "Using %d of %d pixels for gain estimation",
                         N, Ntot);

            gain_mean = sum_vf / N;

            /* 
             * If sigma_i follows a Gaussian distribution and flux_i
             * is roughly constant, then the distribution of  
             *        variance_i / flux_i  =  sigma_i^2 / flux_i
             * has a predictable (non-symmetric and non-Gaussian) shape.
             *
             * We want the *mean* of this distribution, and we cannot use
             * e.g. the median to estimate the mean.
             */

            vfvf_mean = sum_vfvf / N;

            gain_err = sqrt(fabs(vfvf_mean - gain_mean*gain_mean));

            /* 
             * Mean squared is always greater than squared mean,
             * the fabs() is there just in case.
             */

            /* 
             * Convert from stdev to error of mean estimate 
             */

            gain_err /= sqrt(N);
        }
        
        assure( gain_mean > 0, return,
                "Difference between first two raw input frames is "
                "always zero, cannot estimate gain");
        
        conad = 1.0/gain_mean;
        
        /* 
         * Relative error in CONAD and GAIN is the same:
         */

        conad_err = conad * gain_err/gain_mean;
    }
    else {
        cpl_msg_warning(cpl_func, "Only 1 raw frame provided, "
                        "cannot estimate CONAD");
        conad     = -1;
        conad_err = -1;
    }

    fors_qc_write_qc_double(qc,
                            conad,
                            "QC.FLAT.CONAD",
                            "e-/ADU",
                            "Conversion factor from ADU to electrons",
                            setting->instrument);
        
    fors_qc_write_qc_double(qc,
                            conad_err,
                            "QC.FLAT.CONADERR",
                            "e-/ADU",
                            "Error on conversion factor ADU/electrons",
                            setting->instrument);  
#endif

    fors_qc_end_group();

    cleanup;
    return;
}


/**@}*/
