pro amie_pipeline, cfg_file, integer = integer
;+
; NAME:
;   amie_pipeline
;
; PURPOSE:
;   Calibrates AMIE raw images
;
; CATEGORY:
;   AMIE Calibration
;
; CALLING SEQUENCE:
;   amie_pipeline, config_file
;
; INPUTS:
;   config_file - a configuration file, e.g. use 'AMIE_Pipeline.cfg'
;
; OPTIONAL INPUT PARAMETERS:
;   integer - option to switch back to no filter equalization, no
;             deviding by exposure time, and output format integer
;             instead of float.
;
; OUTPUTS:
;    PDS file(s) containing the calibrated images. The files will be written in the
;    directory as defined in the configuration file.
;
; COMMON BLOCKS:
;   None.
;
; SIDE EFFECTS:
;   Hopefully none.
;
; RESTRICTIONS:
;   No restrictions
;
; MODIFICATION HISTORY:
;    2004 Feb 12, ma: Draft version, not tested yet!
;    2008 Apr 25, sm: Create directory structure in alphabetical order
;
;    (c) 2004, Miguel Almeida, ESA/RSSD
;-

;; 2008 Feb 13, BG: Check for keyword integer
if keyword_set( integer ) then begin
    integer = integer
endif else begin
    integer = 0
endelse
if integer ne 0 then integer = 1

;---------------------------------------------------------------------------------------------------------------------------
;
;Initializes the log file
;
;---------------------------------------------------------------------------------------------------------------------------

logname = initialize_log('AMIE_pipeline_')

;---------------------------------------------------------------------------------------------------------------------------
;
;Read configuration file where the the raw data and the final destination directories are defined
;
;---------------------------------------------------------------------------------------------------------------------------

if n_elements(cfg_file) eq 0 then cfg_file = ''

if cfg_file eq '' then begin
    cfg_file = 'amie_pipeline.cfg'
endif
cfg_content = read_amie_pipeline_cfg(cfg_file, logname)
create_log_entry, logname, 'exited configuration reader module (read_amie_pipeline_cfg)'

raw_path = cfg_content(0)
cal_path = cfg_content(1)

;------------------------------------------------------------------------------
; 2008-03-27, BG: Read the master frames now here outside the loop

; master flat frame
master_flat = read_amie(cfg_content(5), flat_label, orientation = 'CCD_FRAME')
create_log_entry, logname, 'Loaded Master Flat Frame'

; master bias frame
sBiasFile = findfile(cfg_content(6) + 'AMI_CMA_*_00000.IMG', $
                     count = iNumberFiles)
if (iNumberFiles ne 1) then begin
   create_log_entry, logname, $
      'Warning: More than one AMI_CMA_*_00000.IMG file found in the directory'
endif
master_bias = read_amie( sBiasFile(iNumberFiles-1), bias_label, $
                         orientation = 'CCD_FRAME' )
create_log_entry, logname, 'Master bias read from file ' + $
                           sBiasFile(iNumberFiles-1)

; master slope frame
sSlopeFile = findfile(cfg_content(6) + 'AMI_CMA_*_00001.IMG', $
                      count = iNumberFiles)
if (iNumberFiles ne 1) then begin
   create_log_entry, logname, $
       'Warning: More than one AMI_CMA_*_00001.IMG file found in the directory'
endif
master_slope = read_amie( sSlopeFile(iNumberFiles-1), slope_label, $
                          orientation = 'CCD_FRAME')
create_log_entry, logname, 'Dark current read from file ' + $
                           sSlopeFile(iNumberFiles-1)

;---------------------------------------------------------------------------------------------------------------------------
;
;Creates a new tree structure for the new images based on the old tree structure
;
;---------------------------------------------------------------------------------------------------------------------------

old_tree = expand_path('+' + raw_path, /all_dirs,/array, count = ndirs)  ;reads in the old tree structure
old_tree = old_tree[sort(old_tree)]  ; alphabetical order 
new_tree = strarr(ndirs)    ;creates an array for the new directories
old_home_len = strlen(raw_path)
for k = 0, ndirs(0)-1 do begin                           ;creates the new tree stucture
    new_tree(k) = cal_path + strmid(old_tree(k), old_home_len)
    file_mkdir, new_tree(k)
endfor

create_log_entry, logname, 'new directories for the calibrated data created'

;---------------------------------------------------------------------------------------------------------------------------
;
;Read all filenames to be calibrated and calibrates one by one
;
;---------------------------------------------------------------------------------------------------------------------------

n_cal_files = 100000 ; create this as long int!
n_cal_files = n_cal_files - n_cal_files
n_skip_files = 0

for l = 0, ndirs(0) - 1 do begin       ;cycles trough all the old directories looking for AMIE files to be calibrated

    filenames = findfile(old_tree(l) + path_sep() + '*.IMG', count = n_files)    ;reads in the filenames in a directory

    if n_files gt 0 then begin

       for i = 0, (n_files - 1) do begin  ;cycles trough all the images in a directory

        ;; 2008-Jan-13, BG: Added orientaton.
        raw_image = read_amie(filenames(i), raw_label, orientation = 'CCD_FRAME')
        if n_elements(raw_label) eq 0 then raw_label = ''
        if (n_elements(raw_image.image1) eq 1 or n_elements(raw_label) eq 1) then begin
          create_log_entry, logname, filenames(i) + ' non existent, incomplete or corrupt'
        endif else begin

          ;; 2008-Feb-21, BG: Skip file if no temperature or no exposure time
          exposure_duration = pdspar(raw_label, 'EXPOSURE_DURATION')
          focal_plane_temperature = pdspar(raw_label, 'FOCAL_PLANE_TEMPERATURE')
          if exposure_duration eq 'UNK' or focal_plane_temperature eq 'UNK' then begin
            create_log_entry, logname, 'skipping ' + filenames(i)
            n_skip_files = n_skip_files + 1
          endif else begin

            create_log_entry, logname, 'calibrating ' + filenames(i)
            
            ;; cal_file_name = pdspar(raw_label, 'FILE_NAME')
            cal_file_name = STRMID(pdspar(raw_label,'FILE_NAME'),1)
   			cal_file_name = STRMID(cal_file_name,0,STRLEN(cal_file_name)-1)
            
            strput, cal_file_name, 'R', 5

            ;; 2008-Jan-13, BG: Added option integer
            ;; 2008-03-27, BG: Added master flat
            calibrated_image = calibrate_amie(raw_label, raw_image, $
               cfg_content=cfg_content, logname=logname, integer=integer, $
               master_flat=master_flat, master_bias=master_bias, $
               master_slope=master_slope)

            calibrated_label = create_new_label(raw_label, filenames(i), cal_file_name, cfg_content, logname,/COMMENTS)
            cal_browse_image = do_browse_image(calibrated_image, raw_image.image0, raw_label, 128, logname)
			      ;;cal_browse_image = raw_image.image0
			
            ;; 2008-Jan-13, BG: Bit shift only if option integer
            ;; given, if not, update the label values for
            ;; 4 byte float output format.
            if integer eq 1 then begin
                sample_bit_mask = pdspar(raw_label, 'SAMPLE_BIT_MASK')
                if sample_bit_mask(1) EQ '"2#1111111111000000#"' then begin
                    calibrated_image = ishft(calibrated_image, +6)
                endif
            endif else begin
                record_bytes = pdspar(calibrated_label, 'RECORD_BYTES')
                browse_records = 16384 / record_bytes
                record_bytes = 2 * record_bytes
                calibrated_label = pdsupdatekeyword(calibrated_label, 'RECORD_BYTES', $
                                   string( record_bytes, format = '(i0)' ) )
                file_records = pdspar(calibrated_label, 'FILE_RECORDS')
                label_records = pdspar(calibrated_label, 'LABEL_RECORDS')
                file_records = file_records - $
                               ( label_records + browse_records ) / 2
                calibrated_label = pdsupdatekeyword(calibrated_label, 'FILE_RECORDS', $
                                   string( file_records, format = '(i0)' ) )
                label_records = label_records / 2
                calibrated_label = pdsupdatekeyword(calibrated_label, 'LABEL_RECORDS', $
                                   string( label_records, format = '(i0)' ) )
                calibrated_label = pdsupdatekeyword(calibrated_label, 'SAMPLE_TYPE', 'PC_REAL', object='IMAGE')
                calibrated_label = pdsupdatekeyword(calibrated_label, 'SAMPLE_BITS', '32', object='IMAGE')
                                ;    calibrated_label = pdsupdatekeyword(calibrated_label, $
                                ;                       'SAMPLE_BIT_MASK', '"N/A"                ', object='IMAGE')
                calibrated_label = pdsupdatekeyword(calibrated_label, $
                                   'SCALING_FACTOR', '1       ', object='IMAGE')
                                ; We now use SCALING_FACTOR instead of
                                ; SAMPLE_BIT_MASK, 2010-Mar-30, BG
            endelse

            ;; 2008-Jan-14, BG: Use computed derived min and max in
            ;; either case, with /integer option or without
            derived_minimum = min( calibrated_image )
            calibrated_label = pdsupdatekeyword(calibrated_label, 'DERIVED_MINIMUM', $
                               string( derived_minimum, format = '(f0)' ) )
            derived_maximum = max( calibrated_image )
            calibrated_label = pdsupdatekeyword(calibrated_label, 'DERIVED_MAXIMUM', $
                               string( derived_maximum, format = '(f0)' ) )

            ;; 200-Feb-21, SM: Change "\" (Windows path separator) to path_sep (OS path separator)
            write_amie, new_tree(l) + path_sep() + cal_file_name, calibrated_image, cal_browse_image, calibrated_label
            create_log_entry, logname, cal_file_name + ' created'
          
            n_cal_files = n_cal_files + 1

          endelse

        endelse

       endfor

    endif

endfor

if n_cal_files eq 0 then begin
    print, 'no files to be calibrated'
    create_log_entry, logname, 'no files to be calibrated'
endif else begin
    if n_cal_files eq 1 then begin
       create_log_entry, logname, strtrim(string(n_cal_files),2) + ' AMIE image calibrated'
       print, strtrim(string(n_cal_files),2) + ' AMIE image calibrated'
    endif else begin
       create_log_entry, logname, strtrim(string(n_cal_files),2) + ' AMIE images calibrated'
       print, strtrim(string(n_cal_files),2) + ' AMIE images calibrated'
    endelse
endelse
create_log_entry, logname, strtrim(string(n_skip_files),2) + ' AMIE images skipped'
print, strtrim(string(n_skip_files),2) + ' AMIE images skipped'

end