/* $Id: fors_sumflux.c,v 1.11 2013-04-24 14:14:13 cgarcia Exp $
 *
 * This file is part of the FORS Data Reduction Pipeline
 * Copyright (C) 2002-2010 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 */

/*
 * $Author: cgarcia $
 * $Date: 2013-04-24 14:14:13 $
 * $Revision: 1.11 $
 * $Name: not supported by cvs2svn $
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <math.h>
#include <cpl.h>
#include <moses.h>
#include <fors_dfs.h>
#include <fors_qc.h>

static int fors_sumflux_create(cpl_plugin *);
static int fors_sumflux_exec(cpl_plugin *);
static int fors_sumflux_destroy(cpl_plugin *);
static int fors_sumflux(cpl_parameterlist *, cpl_frameset *);

static char fors_sumflux_description[] =
"This recipe is used to monitor any lamp flux on the CCD. The input raw\n"
"image should be either a FLUX_ARC_LSS or a FLUX_FLAT_LSS frame. After the\n"
"background subtraction the total signal is integrated and divided by the\n"
"exposure time and by the total number of CCD original pixels (keeping\n"
"into account a possible rebinned readout). In the case of FORS2 frames\n"
"the background is the median level evaluated from the available overscan\n"
"regions. In the case of FORS1 data, where overscan regions are missing,\n"
"the background is evaluated as the median level of the first 200 CCD columns\n"
"for flat field data, while for arc lamp data a background map evaluated\n"
"from the regions without spectral lines is computed and subtracted. The\n"
"background subtracted frame is written to output in all cases, and the QC\n"
"parameters QC LAMP FLUX and QC LAMP FLUXERR are computed.\n\n"
"Input files:\n\n"
"  DO category:      Type:       Explanation:         Required:\n"
"  FLUX_FLAT_LSS     Raw         Flat field exposure     Y\n"
"  or FLUX_ARC_LSS   Raw         Arc lamp exposure       Y\n\n"
"Output files:\n\n"
"  DO category:      Data type:  Explanation:\n"
"  FLUX_LAMP_LSS     FITS image  Background subtracted integration region\n\n";

#define fors_sumflux_exit(message)            \
{                                             \
if ((const char *)message != NULL) cpl_msg_error(recipe, message);  \
cpl_free(instrume);                           \
cpl_free(pipefile);                           \
cpl_image_delete(master_bias);                \
cpl_image_delete(exposure);                   \
cpl_propertylist_delete(header);              \
cpl_propertylist_delete(qclist);              \
cpl_table_delete(overscans);                  \
cpl_msg_indent_less();                        \
return -1;                                    \
}

#define fors_sumflux_exit_memcheck(message)   \
{                                             \
if ((const char *)message != NULL) cpl_msg_info(recipe, message);   \
cpl_free(instrume);                           \
cpl_free(pipefile);                           \
cpl_image_delete(master_bias);                \
cpl_image_delete(exposure);                   \
cpl_propertylist_delete(header);              \
cpl_propertylist_delete(qclist);              \
cpl_table_delete(overscans);                  \
cpl_msg_indent_less();                        \
return 0;                                     \
}


/**
 * @brief    Build the list of available plugins, for this module. 
 *
 * @param    list    The plugin list
 *
 * @return   0 if everything is ok, -1 otherwise
 *
 * Create the recipe instance and make it available to the application 
 * using the interface. This function is exported.
 */

int cpl_plugin_get_info(cpl_pluginlist *list)
{
    cpl_recipe *recipe = cpl_calloc(1, sizeof *recipe );
    cpl_plugin *plugin = &recipe->interface;

    cpl_plugin_init(plugin,
                    CPL_PLUGIN_API,
                    FORS_BINARY_VERSION,
                    CPL_PLUGIN_TYPE_RECIPE,
                    "fors_sumflux",
                    "Integrate flux from all or part of the input frame",
                    fors_sumflux_description,
                    "Carlo Izzo",
                    PACKAGE_BUGREPORT,
    "This file is currently part of the FORS Instrument Pipeline\n"
    "Copyright (C) 2002-2010 European Southern Observatory\n\n"
    "This program is free software; you can redistribute it and/or modify\n"
    "it under the terms of the GNU General Public License as published by\n"
    "the Free Software Foundation; either version 2 of the License, or\n"
    "(at your option) any later version.\n\n"
    "This program is distributed in the hope that it will be useful,\n"
    "but WITHOUT ANY WARRANTY; without even the implied warranty of\n"
    "MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n"
    "GNU General Public License for more details.\n\n"
    "You should have received a copy of the GNU General Public License\n"
    "along with this program; if not, write to the Free Software Foundation,\n"
    "Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA\n",
                    fors_sumflux_create,
                    fors_sumflux_exec,
                    fors_sumflux_destroy);

    cpl_pluginlist_append(list, plugin);
    
    return 0;
}


/**
 * @brief    Setup the recipe options    
 *
 * @param    plugin  The plugin
 *
 * @return   0 if everything is ok
 *
 * Defining the command-line/configuration parameters for the recipe.
 */

static int fors_sumflux_create(cpl_plugin *plugin)
{
    cpl_recipe    *recipe;
    cpl_parameter *p;


    /* 
     * Check that the plugin is part of a valid recipe 
     */

    if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
        recipe = (cpl_recipe *)plugin;
    else 
        return -1;

    /* 
     * Create the parameters list in the cpl_recipe object 
     */

    recipe->parameters = cpl_parameterlist_new(); 


    /*
     * X coordinate of lower left corner
     */

    p = cpl_parameter_new_value("fors.fors_sumflux.xlow",
                                CPL_TYPE_INT,
                                "X coordinate of lower left corner "
                                "of integration region (pixel)",
                                "fors.fors_sumflux",
                                0);
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "xlow");
    cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(recipe->parameters, p);

    /*
     * Y coordinate of lower left corner
     */

    p = cpl_parameter_new_value("fors.fors_sumflux.ylow",
                                CPL_TYPE_INT,
                                "Y coordinate of lower left corner "
                                "of integration region (pixel)",
                                "fors.fors_sumflux",
                                0);
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "ylow");
    cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(recipe->parameters, p);

    /*
     * X coordinate of upper right corner
     */

    p = cpl_parameter_new_value("fors.fors_sumflux.xhigh",
                                CPL_TYPE_INT,
                                "X coordinate of upper right corner "
                                "of integration region (pixel) (0 = CCD size)",
                                "fors.fors_sumflux",
                                0);
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "xhigh");
    cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(recipe->parameters, p);

    /*
     * Y coordinate of upper right corner
     */

    p = cpl_parameter_new_value("fors.fors_sumflux.yhigh",
                                CPL_TYPE_INT,
                                "Y coordinate of upper right corner "
                                "of integration region (pixel) (0 = CCD size)",
                                "fors.fors_sumflux",
                                0);
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "yhigh");
    cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(recipe->parameters, p);

    return 0;
}


/**
 * @brief    Execute the plugin instance given by the interface
 *
 * @param    plugin  the plugin
 *
 * @return   0 if everything is ok
 */

static int fors_sumflux_exec(cpl_plugin *plugin)
{
    cpl_recipe *recipe;
    
    if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
        recipe = (cpl_recipe *)plugin;
    else 
        return -1;

    return fors_sumflux(recipe->parameters, recipe->frames);
}


/**
 * @brief    Destroy what has been created by the 'create' function
 *
 * @param    plugin  The plugin
 *
 * @return   0 if everything is ok
 */

static int fors_sumflux_destroy(cpl_plugin *plugin)
{
    cpl_recipe *recipe;
    
    if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
        recipe = (cpl_recipe *)plugin;
    else 
        return -1;

    cpl_parameterlist_delete(recipe->parameters); 

    return 0;
}


/**
 * @brief    Interpret the command line options and execute the data processing
 *
 * @param    parlist     The parameters list
 * @param    frameset    The set-of-frames
 *
 * @return   0 if everything is ok
 */

static int fors_sumflux(cpl_parameterlist *parlist, cpl_frameset *frameset)
{

    const char *recipe = "fors_sumflux";


    /*
     * Input parameters
     */

    int xlow;
    int ylow;
    int xhig;
    int yhig;

    /*
     * CPL objects
     */

    cpl_image        *master_bias = NULL;
    cpl_image        *exposure    = NULL;
    cpl_image        *dummy       = NULL;

    cpl_table        *overscans   = NULL;

    cpl_propertylist *header      = NULL;
    cpl_propertylist *qclist      = NULL;

    /*
     * Auxiliary variables
     */

    const char   *arc_tag  = "FLUX_ARC_LSS";
    const char   *flat_tag = "FLUX_FLAT_LSS";

    char    version[80];
    char    lamp[20];
    const char   *exposure_tag;
    const char   *flux_tag = "FLUX_LAMP_LSS";
    double  time;
    double  norm_factor;
    int     nframes;
    int     rebin;
    int     nx, ny;
    double  gain;
    double  flux, flux_err;
    int     i;

    char   *instrume = NULL;
    char   *pipefile = NULL;


    snprintf(version, 80, "%s-%s", PACKAGE, PACKAGE_VERSION);

    cpl_msg_set_indentation(2);

    /* 
     * Get configuration parameters
     */

    cpl_msg_info(recipe, "Recipe %s configuration parameters:", recipe);
    cpl_msg_indent_more();

    xlow = dfs_get_parameter_int(parlist, "fors.fors_sumflux.xlow", NULL);
    ylow = dfs_get_parameter_int(parlist, "fors.fors_sumflux.ylow", NULL);
    xhig = dfs_get_parameter_int(parlist, "fors.fors_sumflux.xhigh", NULL);
    yhig = dfs_get_parameter_int(parlist, "fors.fors_sumflux.yhigh", NULL);

    if (cpl_error_get_code())
        fors_sumflux_exit("Failure getting the configuration parameters");

    if (xlow > xhig || ylow > yhig || xhig < 0 || yhig < 0)
        fors_sumflux_exit("Invalid integration region");

    
    /* 
     * Check input set-of-frames
     */

    cpl_msg_indent_less();
    cpl_msg_info(recipe, "Check input set-of-frames:");
    cpl_msg_indent_more();

    if (!dfs_equal_keyword(frameset, "ESO DET CHIP1 ID")) 
        cpl_msg_warning(cpl_func,"Input frames are not from the same chip");

    nframes = cpl_frameset_count_tags(frameset, arc_tag)
            + cpl_frameset_count_tags(frameset, flat_tag);

    if (nframes == 0)
        fors_sumflux_exit("Missing input LSS calibration exposures");

    if (nframes > 1) {
        cpl_msg_error(recipe, "Too many LSS calibration exposures found (%d). "
                      "Just one is required.", nframes);
        fors_sumflux_exit(NULL);
    }

    if (cpl_frameset_count_tags(frameset, arc_tag) > 0)
        exposure_tag = arc_tag;
    else
        exposure_tag = flat_tag;

/*** MASTER BIAS

    if (cpl_frameset_count_tags(frameset, "MASTER_BIAS") == 0)
        fors_sumflux_exit("Missing required input: MASTER_BIAS");

    if (cpl_frameset_count_tags(frameset, "MASTER_BIAS") > 1)
        fors_sumflux_exit("Too many in input: MASTER_BIAS");

    cpl_msg_info(recipe, "Load master bias frame...");
    cpl_msg_indent_more();

    master_bias = dfs_load_image(frameset, "MASTER_BIAS", CPL_TYPE_FLOAT, 0, 1);
    if (master_bias == NULL)
        fors_sumflux_exit("Cannot load master bias");

MASTER BIAS ***/

    cpl_msg_indent_less();
    cpl_msg_info(recipe, "Load %s frame...", exposure_tag);
    cpl_msg_indent_more();

    exposure = dfs_load_image(frameset, exposure_tag, CPL_TYPE_FLOAT, 0, 0);
    if (exposure == NULL)
        fors_sumflux_exit("Cannot load input frame");

    /*
     * Get exposure time, rebin factor, gain, etc.
     */

    header = dfs_load_header(frameset, exposure_tag, 0);

    if (header == NULL)
        fors_sumflux_exit("Cannot load input frame header");

    time = cpl_propertylist_get_double(header, "EXPTIME");

    if (cpl_error_get_code() != CPL_ERROR_NONE)
        fors_sumflux_exit("Missing keyword EXPTIME in input frame header");

    instrume = (char *)cpl_propertylist_get_string(header, "INSTRUME");
    if (instrume == NULL)
        fors_sumflux_exit("Missing keyword INSTRUME in input frame header");
    instrume = cpl_strdup(instrume);

    if (instrume[4] == '1')
        snprintf(version, 80, "%s/%s", "fors1", VERSION);
    if (instrume[4] == '2')
        snprintf(version, 80, "%s/%s", "fors2", VERSION);

    rebin = cpl_propertylist_get_int(header, "ESO DET WIN1 BINX");

    if (cpl_error_get_code() != CPL_ERROR_NONE)
        fors_sumflux_exit("Missing keyword ESO DET WIN1 BINX in input "
                          "frame header");

    rebin *= cpl_propertylist_get_int(header, "ESO DET WIN1 BINY");

    if (cpl_error_get_code() != CPL_ERROR_NONE)
        fors_sumflux_exit("Missing keyword ESO DET WIN1 BINY in input "
                          "frame header");

    if (rebin > 1) {
        cpl_msg_info(recipe, 
                     "One readout pixel corresponds to %d chip pixels", rebin);
    }

    gain = cpl_propertylist_get_double(header, "ESO DET OUT1 CONAD");

    if (cpl_error_get_code() != CPL_ERROR_NONE)
        fors_sumflux_exit("Missing keyword ESO DET OUT1 CONAD in arc lamp "
                        "frame header");

    cpl_msg_info(recipe, "The gain factor is: %.2f e-/ADU", gain);

    /* Leave the header on for the next step... */


    /*
     * Remove the bias if possible (FORS2), otherwise remove the flux from a 
     * presumably darker part of the image (FORS1).
     */

    switch (instrume[4]) {
    case '1':
#ifdef OLD_FORS1
        cpl_msg_info(recipe, "Remove low-flux region level...");
        if (exposure_tag == flat_tag) {
            overscans = cpl_table_new(2);
            cpl_table_new_column(overscans, "xlow", CPL_TYPE_INT);
            cpl_table_new_column(overscans, "ylow", CPL_TYPE_INT);
            cpl_table_new_column(overscans, "xhig", CPL_TYPE_INT);
            cpl_table_new_column(overscans, "yhig", CPL_TYPE_INT);
    
            nx = cpl_image_get_size_x(exposure);
            ny = cpl_image_get_size_y(exposure);
    
            /* "Valid" region */
    
            cpl_table_set_int(overscans, "xlow", 0, 200);
            cpl_table_set_int(overscans, "ylow", 0, 0);
            cpl_table_set_int(overscans, "xhig", 0, nx);
            cpl_table_set_int(overscans, "yhig", 0, ny);
    
            /* "Overscan" (background) region */
    
            cpl_table_set_int(overscans, "xlow", 1, 0);
            cpl_table_set_int(overscans, "ylow", 1, 0);
            cpl_table_set_int(overscans, "xhig", 1, 200);
            cpl_table_set_int(overscans, "yhig", 1, ny);
        }
        else {
            background = mos_arc_background(exposure, 15, 15);
            cpl_image_subtract(exposure, background);
            cpl_image_delete(background);
        }
#else
        cpl_msg_info(recipe, "Remove bias, evaluated on overscan regions...");
        overscans = mos_load_overscans_vimos(header, 1);
#endif
        break;
    case '2':
        cpl_msg_info(recipe, "Remove bias, evaluated on overscan regions...");
        overscans = mos_load_overscans_vimos(header, 1);
        break;
    default:
        cpl_msg_error(recipe, "Invalid instrument name: %s", instrume);
        fors_sumflux_exit(NULL);
    }

    if (overscans) {
        dummy = mos_remove_bias(exposure, NULL, overscans);
        cpl_table_delete(overscans); overscans = NULL;
        cpl_image_delete(exposure); exposure = dummy;
    
        if (exposure == NULL)
            fors_sumflux_exit("Cannot remove bias from input frame");
    }

    nx = cpl_image_get_size_x(exposure);
    ny = cpl_image_get_size_y(exposure);

    if (xhig == 0)
        xhig = nx;

    if (yhig == 0)
        yhig = ny;

    if (xlow > nx || ylow > ny || xhig < 0 || yhig < 0)
        fors_sumflux_exit("The integration region lays outside the CCD");

    if (xlow == xhig || ylow == yhig)
        fors_sumflux_exit("The integration area is zero");

    norm_factor = rebin * time * (xhig - xlow) * (yhig - ylow);

    flux = cpl_image_get_flux(exposure);
    if (flux > 0.0) {
        flux_err = sqrt(flux/gain);
    }
    else {
        flux = 0.0;
        flux_err = 0.0;
    }

    flux /= norm_factor;
    flux_err /= norm_factor;

    cpl_msg_info(recipe, "Flux: %.4f +/- %.4f (ADU/s*pixel)", flux, flux_err);

    cpl_image_divide_scalar(exposure, norm_factor);

    /* Leave the header on for the next step... */


    /*
     * QC1 group header
     */

    qclist = cpl_propertylist_new();
    
    fors_qc_start_group(qclist, "2.0", instrume);

    if (fors_qc_write_string("PRO.CATG", flux_tag,
                            "Product category", instrume))
        fors_sumflux_exit("Cannot write product category to QC log file");

    if (fors_qc_keyword_to_paf(header, "ESO DPR TYPE", NULL,
                              "DPR type", instrume))
        fors_sumflux_exit("Missing keyword DPR TYPE in frame header");

    if (fors_qc_keyword_to_paf(header, "ESO TPL ID", NULL,
                              "Template", instrume))
        fors_sumflux_exit("Missing keyword TPL ID in frame header");

    if (fors_qc_keyword_to_paf(header, "ESO INS GRIS1 NAME", NULL,
                              "Grism name", instrume))
        fors_sumflux_exit("Missing keyword INS GRIS1 NAME in frame header");

    if (fors_qc_keyword_to_paf(header, "ESO INS GRIS1 ID", NULL,
                              "Grim identifier", instrume))
        fors_sumflux_exit("Missing keyword INS GRIS1 ID in frame header");

    if (cpl_propertylist_has(header, "ESO INS FILT1 NAME"))
        fors_qc_keyword_to_paf(header, "ESO INS FILT1 NAME", NULL,
                              "Filter name", instrume);

    if (fors_qc_keyword_to_paf(header, "ESO INS COLL NAME", NULL,
                              "Collimator name", instrume))
        fors_sumflux_exit("Missing keyword INS COLL NAME in frame header");

    if (fors_qc_keyword_to_paf(header, "ESO DET CHIP1 ID", NULL,
                              "Chip identifier", instrume))
        fors_sumflux_exit("Missing keyword DET CHIP1 ID in frame header");

    if (fors_qc_keyword_to_paf(header, "ESO INS SLIT WID",
                              "arcsec", "Slit width", instrume))
        fors_sumflux_exit("Missing keyword ESO INS SLIT WID in frame header");

    if (fors_qc_keyword_to_paf(header, "ESO DET OUT1 CONAD", "e-/ADU", 
                              "Conversion from ADUs to electrons", instrume))
        fors_sumflux_exit("Missing keyword ESO DET OUT1 CONAD in frame header");

    if (fors_qc_keyword_to_paf(header, "ESO DET WIN1 BINX", NULL, 
                              "Binning factor along X", instrume))
        fors_sumflux_exit("Missing keyword ESO DET WIN1 BINX in frame header");

    if (fors_qc_keyword_to_paf(header, "ESO DET WIN1 BINY", NULL, 
                              "Binning factor along Y", instrume))
        fors_sumflux_exit("Missing keyword ESO DET WIN1 BINY in frame header");

    for (i = 1; i < 7; i++) {
       snprintf(lamp, 20, "ESO INS LAMP%d NAME", i);
       if (cpl_propertylist_has(header, lamp))
            fors_qc_keyword_to_paf(header, lamp, NULL,
                                  "Name of lamp on", instrume);
    }

    if (fors_qc_keyword_to_paf(header, "ARCFILE", NULL,
                              "Archive name of input data", instrume))
        fors_sumflux_exit("Missing keyword ARCFILE in frame header");

    cpl_propertylist_delete(header); header = NULL;

    pipefile = dfs_generate_filename(flux_tag);
    if (fors_qc_write_string("PIPEFILE", pipefile,
                            "Pipeline product name", instrume))
        fors_sumflux_exit("Cannot write PIPEFILE to QC log file");
    cpl_free(pipefile); pipefile = NULL;


    /*
     * QC1 parameters
     */

    if (fors_qc_write_qc_double(qclist, flux, "QC.LAMP.FLUX", "ADU/s*pixel", 
                               "Total lamp flux", instrume)) {
        fors_sumflux_exit("Cannot write total lamp flux to QC log file");
    }
    
    if (fors_qc_write_qc_double(qclist, flux_err, "QC.LAMP.FLUXERR", 
                               "ADU/s*pixel", 
                               "Error on lamp flux", instrume)) {
        fors_sumflux_exit("Cannot write error on lamp flux to QC log file");
    }

    fors_qc_end_group();

    cpl_free(instrume); instrume = NULL;

    if (dfs_save_image(frameset, exposure, flux_tag, qclist,
                       parlist, recipe, version))
        fors_sumflux_exit(NULL);

    cpl_image_delete(exposure); exposure = NULL;
    cpl_propertylist_delete(qclist); qclist = NULL;

    return 0;

} 
