/***************************************
  $Revision: 1.55 $

  Functions to process data stream( file, network socket, etc.)

  Status: NOT REVUED, NOT TESTED

 Author(s):       Chris Ottrey, Andrei Robachevsky

  ******************/ /******************
  Modification History:
        andrei (17/01/2000) Created.
  ******************/ /******************
  Copyright (c) 2000,2001                         RIPE NCC
 
  All Rights Reserved
  
  Permission to use, copy, modify, and distribute this software and its
  documentation for any purpose and without fee is hereby granted,
  provided that the above copyright notice appear in all copies and that
  both that copyright notice and this permission notice appear in
  supporting documentation, and that the name of the author not be
  used in advertising or publicity pertaining to distribution of the
  software without specific, written prior permission.
  
  THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING
  ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS; IN NO EVENT SHALL
  AUTHOR BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY
  DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
  AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 ***************************************/

#include "rip.h"

#include <sys/types.h>
#include <sys/socket.h>
#include <netdb.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <string.h>

 
typedef enum _Line_Type_t {
 LINE_ATTRIBUTE,
 LINE_COMMENT,
 LINE_EMPTY,
 LINE_EOF,
 LINE_ADD,
 LINE_UPD,
 LINE_DEL,
 LINE_OVERRIDE_ADD,
 LINE_OVERRIDE_UPD,
 LINE_OVERRIDE_DEL,
 LINE_ACK
} Line_Type_t;

/* Maximum number of objects(serials) we can consume at a time */
#define SBUNCH 1000

static int report_transaction(Transaction_t *tr, long transaction_id, Log_t *log, ut_timer_t *psotime, char *reason);
static int process_nrtm(UD_stream_t *ud_stream, Transaction_t *tr, int operation);
static int process_updates(UD_stream_t *ud_stream, Transaction_t *tr, int operation);
static int process_transaction(UD_stream_t *ud_stream, Obj_parse_t *parse, int operation, long transaction_id);

static GSList *ud_split_attribute(GSList *attr_list, Attribute_t *attr);
static void ud_parse_init(SQ_connection_t *sql_connection, Obj_parse_t *parse);
static void ud_parse_free(Obj_parse_t *parse);
static int line_continuation(char *line);
static GString *escape_apostrophes(GString *text);
static Object_t *UD_parse_object(Obj_parse_t *parse, char *line_buff);
static int ud_replace_substring(GString *gstring, char *old, char *new);
static int ud_normalize(Obj_parse_t *parse);
                                                                                                
/* Delimiters that separate list members, both RPS(,) and legacy( ) */
#define ATTR_DELIMITERS " ,\t"


static void ud_parse_init(SQ_connection_t *sql_connection, Obj_parse_t *parse){
     bzero(parse, sizeof(Obj_parse_t));
     parse->garbage = 0;
     parse->sql_connection = sql_connection;     
}

static void ud_parse_free(Obj_parse_t *parse){
     UT_free(parse->nic);	
     UT_free(parse->object_name);
}


/******************************************************************
*  int line_continuation()                                        *
*                                                                 *
*  Checks if the line starts with one of line continuation signs  *
*  Returns 1 if this is line continuation, 0 otherwise            *
*                                                                 *
* ****************************************************************/
static int line_continuation(char *line)
{
 switch(*line) {
	case ' ':
	case '\t':
	case '+':
	         return(1); /* these indicate line continuation */
	default: return(0); 
 }

}

/******************************************************************
*  int ud_replace_substring()                                     *
*                                                                 *
*  replace a subsring in a string                                 *
*  Returns -1 if no match was found                               *
*                                                                 *
* ****************************************************************/
static int ud_replace_substring(GString *gstring, char *old, char *new)
{
 int i, i0=-1, j=0;
 
	for (i=0; ((i < gstring->len) && (old[j] != '\0')); i++){
		if(gstring->str[i]==old[j]){
			if(i0==-1)i0=i;
			j++;
		} else {
			i0=-1;
			j=0;
		}
	}
	/* if we haven't reached the end of the old substring */
	/* or no match occured return -1 */
	if((old[j] != '\0') || (i0==-1)) return (-1);
	g_string_erase(gstring, i0, j);
	g_string_insert(gstring, i0, new);
	return(0);
}

/************************************************************
*                                                           *
* The function to splits attribute value into multiple      *
* words and appends them as attr_type - attr_valu pairs     *
* to the attr_list                                          *
*                                                           *
*                                                           *
************************************************************/
static GSList *ud_split_attribute(GSList *attr_list, Attribute_t *attr)
{
 char *token;
 char *split;
 char *value, *n;
 Attribute_t *attr_split;
 GSList *the_list = attr_list;


  /* check for line continuation (+) */
  if (strncmp(attr->value, "+", 1) == 0) attr->value++;
  /* check for end-of-line comments */
  n = index(attr->value, '#');
      /* if there is no comment check for trailing \n */
  if(n == NULL) n = index(attr->value, '\n');
  /* now copy the clean value into the attribute */
  if(n == NULL) value = g_strdup(attr->value); 
  else  value = g_strndup(attr->value, (n - attr->value));
     
  token=value;
  while((split=strsep(&token, ATTR_DELIMITERS))){
    if (*split != '\0'){	  
      attr_split = attribute_new1(attr->type, split);
      if (attr_split) the_list = g_slist_append(the_list, attr_split);
    }
  }
  UT_free(value);
  attribute_free(attr, NULL);

 return(the_list);
}

/* XXX */
/* commented out because it is unused - shane
static void each_attribute_print(void *element_data, void *tr_ptr)
{

Attribute_t *attr = (Attribute_t *)element_data;

 fprintf(stderr, "[%d|%s]\n", attr->type, attr->value);

}
*/

/* XXX */
/* commented out because it is unused - shane
static void print_object(Object_t *obj)
{
  g_slist_foreach(obj->attributes, each_attribute_print, NULL);	
  fprintf(stderr, ">>>>>\n%s\n", obj->object->str);
}
*/


/******************************************************************
* GString *escape_apostrophes()                                   *
* Escapes apostrophes in the text so they do not confuse printf   *
* functions and don't corrupt SQL queries                         *
*                                                                 *
* *****************************************************************/
static GString *escape_apostrophes(GString *text) {
  int i;
  for (i=0; i < text->len; i++) {
    if ((text->str[i] == '\'') || (text->str[i] == '\\')) {
      text = g_string_insert_c(text, i, '\\');
      i++;
    }
  }
 return(text); 
} /* escape_apostrophes() */


/******************************************************************
* Line_Type_t line_type(e)                                        *
* Determines the line type analysing the first letters            *
*                                                                 *
* ****************************************************************/
static Line_Type_t line_type(const char *line, long *transaction_id) {

  if (strncmp(line, "# EOF", 4) == 0) return(LINE_EOF);
  if (strncmp(line, "#", 1) == 0)     return(LINE_COMMENT);
  if (strcmp(line, "\n") == 0)        return(LINE_EMPTY);
 
  if (strncmp(line, "ACK", 3) == 0) {
    *transaction_id = atol(line+3);	  
    return(LINE_ACK);
  }
  if (strncmp(line, "ADD_OVERRIDE", 12) == 0) {
    *transaction_id = atol(line+12);	  
    return(LINE_OVERRIDE_ADD);
  }
  if (strncmp(line, "UPD_OVERRIDE", 12) == 0) {
    *transaction_id = atol(line+12);	  
    return(LINE_OVERRIDE_UPD);
  }
  if (strncmp(line, "DEL_OVERRIDE", 12) == 0) {
    *transaction_id = atol(line+12);	  
    return(LINE_OVERRIDE_DEL);
  }
 
  if (strncmp(line, "ADD", 3) == 0) {
    *transaction_id = atol(line+3);
    return(LINE_ADD);
  }
  if (strncmp(line, "UPD", 3) == 0) {
    *transaction_id = atol(line+3);
    return(LINE_UPD);
  }
  if (strncmp(line, "DEL", 3) == 0) {
    *transaction_id = atol(line+3); 
    return(LINE_DEL);
  }
 
/* Otherwise this is an attribute */  
    return(LINE_ATTRIBUTE);

} /* line_type() */

/******************************************************************
* Object_t *UD_parse_object()                                     *
*                                                                 *
* Parses the object accepting line by line                        *
* Stores the object and list of attributes in Obj_parse_t struct  *
*                                                                 *
* ****************************************************************/
static Object_t *UD_parse_object(Obj_parse_t *parse, char *line_buff)
{
GString *g_line_buff;
Attribute_t *attr;
 
  /* in case of garbage */
  if(parse->garbage) return(NULL);
  
  if(parse->obj==NULL) {
  /* which means this is the start - create a new object*/
	  parse->obj = object_new(line_buff);
	  /* if this is not a class attribute - this is a garbage till the next blank line */
	  if(parse->obj==NULL) {
		  parse->garbage = 1; 
		  return(NULL);
	  }
  }
  
  /* allocate buffer or the string */	  
  if ((g_line_buff = g_string_sized_new(STR_XXL)) == NULL){ 
      ER_perror(FAC_UD, UD_MEM, "cannot allocate gstring");
      die; 
  }
  /* copy the input */
  g_string_sprintf(g_line_buff, "%s", line_buff);
  /* escape apostrophes in the input line */
  g_line_buff=escape_apostrophes(g_line_buff);
   
  attr = attribute_new(g_line_buff->str);
  if (attr){
    /* save this attribute for future line continuations */
    /* skip attributes that are not stored in SQL database */	  
    if(DF_get_update_query_type(attr->type)==UD_NULL_){
     parse->current_attr = NULL;
     attribute_free(attr, NULL);
    }
    else {
     parse->current_attr = attr;	    
     if((attr->type == A_MB) || (attr->type == A_NH)) {
        /* insert it at the beginning */
	/* mnt-by should go before member-of to allow correct membership autorization (still done in RIPupd) */
        /* nic-hdl should go before any admin-c, tech-c to prevent errrors in self referencing role objects  */
	parse->obj->attributes = g_slist_insert(parse->obj->attributes, parse->current_attr, 1);
     } else {
        /* append it to the attribute list */
        parse->obj->attributes = g_slist_append(parse->obj->attributes, parse->current_attr);
     }
    }
  }
  /* if this is not a valid attribute, then this may be unknown attr or line continuation */
  else 
   if(line_continuation(g_line_buff->str)) 
   {
    /* if parse->attr == NULL, then this attr is not stored in SQL database - skip it */
    if(parse->current_attr){
    /* this is continuation of a previous attribute */	  
     char *line_value, *new_value;
     char *n;  
     /* normalize and append to current_attr->value */
     /* check for end-of-line comments */
     n = index(g_line_buff->str, '#');
     /* if there is no comment check for trailing \n */
     if(n == NULL) n = index(g_line_buff->str, '\n');
     /* now copy the clean value into the attribute */
     if(n == NULL) line_value = g_strdup(g_line_buff->str); 
     else  line_value = g_strndup(g_line_buff->str, (n - g_line_buff->str));
     /* replace line continuation character with ' ' */
     *line_value = ' ';
     new_value = g_strconcat(parse->current_attr->value, line_value, NULL);
     parse->current_attr = attribute_upd(parse->current_attr, parse->current_attr->type, new_value);
     UT_free(new_value);
     UT_free(line_value); 
    }
   }
  /* otherwise, this is unknown attribute, so we need to break the continuation */
   else parse->current_attr=NULL;
  /* copy the line into object no matter whether it is an attribute or not (continualtion, etc.) */
  g_string_sprintfa(parse->obj->object, "%s", g_line_buff->str);
  g_string_free(g_line_buff, TRUE);
  return(parse->obj);
}  

/******************************************************************
* int each_attribute_2_pass()                                     *
*                                                                 *
* Assigns nic-handles in case of AUTO                             *
* Splits attributes                                               *
*                                                                 *                                                                 *
******************************************************************/
void each_attribute_2_pass(void *element_data, void *ptr)
{
Attribute_t *attr = (Attribute_t *)element_data;
Obj_parse_t *parse = (Obj_parse_t *)ptr;


 /* Strip the white space */
 g_strstrip(attr->value);

 switch(attr->type){
   case A_NH: /* nic-hdl */
          /*  Parse the string into nh structure */
          /*  In case of an AUTO NIC handle check the ID in the database */
          /* Possible errors leave to core processing */
          if(NH_parse(attr->value, &parse->nh_ptr) == 0) { 
           /* this is an AUTO nic handle */
/*           ER_dbg_va(FAC_UD, ASP_UD_OBJ, "%s parsing nic handle: [%s]", UD_TAG, attr->value);*/
           /* Check if we can allocate it */  
            if(NH_check(parse->nh_ptr, parse->sql_connection)>0){
            /* Convert nh to the database format */  
              parse->nic = NH_convert(parse->nh_ptr);
/*           ER_dbg_va(FAC_UD, ASP_UD_OBJ, "%s parsed and converted nic handle: [%s]", UD_TAG, nic); */
              /* replace the auto nic with a real one */
	      if(ud_replace_substring(parse->obj->object, attr->value, parse->nic)==-1) {
		     ER_perror(FAC_UD, UD_BUG, "cannot replace auto nic handle"); 
		     die;
	      }
	      /* Update the attribute */
              attr = attribute_upd(attr, attr->type, parse->nic);
	      
            }
          }
            parse->new_attr_list = g_slist_append(parse->new_attr_list, attr);
            break;
   case A_PN: /* person */
   case A_RO: /* role */
   case A_MR: /* mbrs-by-ref */
   case A_MB: /* mnt-by */
   case A_ML: /* mnt-lower */
   case A_CT: /* cross-mnt */
   case A_MO: /* member-of */
   case A_SD: /* sub-dom */
   case A_RZ: /* rev-srv */
   case A_NS: /* nserver */
            parse->new_attr_list = ud_split_attribute(parse->new_attr_list, attr);
	    break;
   default:
            parse->new_attr_list = g_slist_append(parse->new_attr_list, attr);
	    break;
 }

}

		  
/******************************************************************
* int ud_normalize()                                              *
*                                                                 *
* function accepts Obj_parse_t structure with attribute list      *
*                                                                 *
* It makes another pass to normalize object                       *
* Normalization includes:                                         *
*                                                                 *
* 1) reorder attributes                                           *
*  . mnt-by should go before member-of to allow correct           * 
*      membership autorization (still done in RIPupd)             *
*  . nic-hdl should go before any admin-c, tech-c to prevent      * 
*       errrors in self referencing role objects                  *
* 2) in case of pn/ro check nic handle and replace it in          *
*     the obj->object if needed (AUTO)                            * 
* 3) split some attributes that allow lists                       *
******************************************************************/
static int ud_normalize(Obj_parse_t *parse)
{
GSList *old_attr_list;
GSList *first_element;
Attribute_t *class_attr;

	old_attr_list = parse->obj->attributes;
	/* get class attribute - the first one */
	first_element = g_slist_nth(old_attr_list, 0);
	class_attr = (Attribute_t *)first_element->data;
	/* save object name for reporting and checking ref integrity for names (legacy) */ 	
	parse->object_name = g_strdup(class_attr->value);
	/* reorder the attributes mnt-by and nic-hdl come first */
	/*g_slist_foreach(old_attr_list, each_attribute_1_pass, parse); 
	g_slist_free(old_attr_list);
	old_attr_list = parse->new_attr_list; */
	parse->new_attr_list = NULL;
	/* assign auto nic-hdl and slit attributes */
	g_slist_foreach(old_attr_list, each_attribute_2_pass, parse);
	g_slist_free(old_attr_list);
	parse->obj->attributes = parse->new_attr_list;

	return(1);
}


/******************************************************************
* report_transaction()                                            *
*                                                                 * 
* Prints error report to the log                                  *
*                                                                 *
* reason - additional message that will be included               *
*                                                                 *
* *****************************************************************/
static int report_transaction(Transaction_t *tr, long transaction_id,  Log_t *log, ut_timer_t *psotime, char *reason)
{
int result=0;
ut_timer_t fotime;
float timediff;
const char *class_name = DF_class_type2name(tr->class_type);
char *primary_key = tr->K->str;


 /* calculate statistics */
  UT_timeget(&fotime);
  timediff = UT_timediff(psotime, &fotime);
 
 if(tr->succeeded==0) {
  result=tr->error;
  log->num_failed++;
  ER_inf_va(FAC_UD, ASP_UD_UPDLOG, "[%ld] %.2fs FAILED [%s:%s][%s]", transaction_id, timediff, class_name, primary_key, reason);
/*  ER_inf_va(FAC_UD, ASP_UD_UPDLOG, "[%ld] object: FAILED [%s][%s](%d/%d)", transaction_id, , reason, log->num_failed, (log->num_failed)+(log->num_ok)); */
  if(result & ERROR_U_OBJ) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: referential integrity error", transaction_id);
  if(result & ERROR_U_AUT) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: authentication error", transaction_id);
  if(result & ERROR_U_BADOP) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: unsupported operation", transaction_id);
  if(result & ERROR_U_COP) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: conflicting operation", transaction_id);
  if(result & ERROR_U_NSUP) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: this type is not supported", transaction_id);
  ER_dbg_va(FAC_UD, ASP_UD_OBJ, "[%ld] object: details [%s]", transaction_id, (tr->error_script)->str);
  result=1; /* # of failures */
 }
 else {
  result=0;
  log->num_ok++;
/*  ER_inf_va(FAC_UD, ASP_UD_OBJ, "[%ld] object: OK     [%s](%d/%d)", transaction_id, obj_name, log->num_ok, (log->num_failed)+(log->num_ok)); */
  ER_inf_va(FAC_UD, ASP_UD_OBJ, "[%ld] %.2fs OK     [%s:%s][%s]", transaction_id, timediff, class_name, primary_key, reason);
  ER_dbg_va(FAC_UD, ASP_UD_OBJ, "[%ld] object: details [%s]", transaction_id, (tr->error_script)->str);
 }
                                                                                                                                               
 return(result);
}/* report_transaction() */



/************************************************************
* process_nrtm()                                            *
*                                                           *
* Process object in NRTM client mode                        *
*                                                           *
* nrtm - pointer to _nrtm structure                         *
* log - pointer to Log_t structure                          *
* object_name - name of the object                          * 
* operation - operation code (OP_ADD/OP_DEL)                *
*                                                           *
* Returns:                                                  *
* 1  - okay                                                 *
* <0 - error                                                *
*                                                           *
************************************************************/

static int process_nrtm(UD_stream_t *ud_stream, Transaction_t *tr, int operation)
{
int result=0;
int dummy=0;
struct _nrtm *nrtm = ud_stream->nrtm;
long serial_id;
Log_t *log_ptr= &(ud_stream->log);
ut_timer_t sotime;
int ta_upd_nhr;

    /* Start timer for statistics */
    UT_timeget(&sotime);

  /* We allow NRTM updates for some inconsistent objects                  */
  /* One of the examples is reference by name which looks like nic-handle */
  /* For this purpose we allow dummy creation when updating an object     */
  /* We also check for dummy allowance when deleting an object            */
  /* this is done to allow deletion of person objects referenced by name  */

  tr->mode|=B_DUMMY;
  if(IS_NO_NHR(tr->mode))ta_upd_nhr=0; else ta_upd_nhr = TA_UPD_NHR;
  
  switch (operation) {
  
  case OP_ADD:
    if(nrtm->tr){ /* DEL ADD => saved*/
      if(tr->object_id==0) { 
	/* object does not exist in the DB */      
	/* delete the previous(saved) object*/
        object_process(nrtm->tr); 
        /* create DEL serial */
	UD_lock_serial(nrtm->tr);
	serial_id = UD_create_serial(nrtm->tr);
	CP_CREATE_S_PASSED(nrtm->tr->action); TR_update_status(nrtm->tr); 
	UD_commit_serial(nrtm->tr);
	UD_unlock_serial(nrtm->tr);
        /* Mark TR as clean */
	TR_mark_clean(nrtm->tr);
        /* log the transaction */
	result=report_transaction(nrtm->tr, serial_id, log_ptr, &sotime, "M:DEL");
        
        object_free(nrtm->tr->object);
        transaction_free(nrtm->tr); nrtm->tr=NULL;
        
	/* Create an object and update NHR */
        tr->action=(TA_CREATE | ta_upd_nhr);
	/* restart the timer for statistics */
	UT_timeget(&sotime);
        object_process(tr); /* create a new one*/
	/* create ADD serial */
        UD_lock_serial(tr);
	serial_id = UD_create_serial(tr); 
	CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
	UD_commit_serial(tr);
	UD_unlock_serial(tr);
	/* Mark TR as clean */
	TR_mark_clean(tr);
        /* log the transaction */
	result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD");
      }
      else { 
      /* object already exists in the DB - update or dummy replacement*/
        /*compare the two, may be we may collapse operations*/
        if(tr->object_id==nrtm->tr->object_id) {
          /* DEL-ADD ->> UPDATE */ 
	  object_free(nrtm->tr->object);
          transaction_free(nrtm->tr); nrtm->tr=NULL;
          tr->action=TA_UPD_CLLPS;
          object_process(tr);
	  /* create DEL+ADD serial records */
	  UD_lock_serial(tr);
	  tr->action=TA_DELETE; serial_id = UD_create_serial(tr);
	  result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:DEL(UPD)");

	  /* restart the timer for statistics */
          UT_timeget(&sotime);
	  tr->sequence_id++;
          tr->action=TA_CREATE; serial_id = UD_create_serial(tr);
	  CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
	  UD_commit_serial(tr);
	  UD_unlock_serial(tr);
	  /* Mark TR as clean */
	  TR_mark_clean(tr);
	  result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD(UPD)");
        }
        else { /* this should be a dummy object in the database(that we are going to replace with the real one */
        /* or an interleaved operation*/
          object_process(nrtm->tr); /* delete the previous(saved) object*/
          /* create a DEL serial record */
	  UD_lock_serial(nrtm->tr);
	  serial_id = UD_create_serial(nrtm->tr); 
	  CP_CREATE_S_PASSED(nrtm->tr->action); TR_update_status(nrtm->tr);
	  UD_commit_serial(nrtm->tr);
	  UD_unlock_serial(nrtm->tr);
	  /* Mark TR as clean */
	  TR_mark_clean(nrtm->tr);
/*	  result=report_transaction(nrtm->tr, log_ptr, nrtm->object_name, "NRTM:DEL:While deleting previous(saved)");*/
          /* log the transaction */
	  result=report_transaction(nrtm->tr, serial_id, log_ptr, &sotime, "M:DEL");


          object_free(nrtm->tr->object);
          transaction_free(nrtm->tr); nrtm->tr=NULL;

	  /* restart the timer for statistics */
          UT_timeget(&sotime);
	  
          tr->action=TA_UPDATE;
          /* check if we are replacing a dummy object */
	  dummy=isdummy(tr);
          /* If we are replacing dummy with a real object update NHR */
/*          if(dummy==1) tr->action = (ta_upd_nhr | TA_UPD_DUMMY); */
          if(dummy==1) tr->action = TA_UPD_DUMMY;
/*        fprintf(stderr,"UPDATE next(dummy)\n"); */
          object_process(tr); /* create a new one*/
/*          result=report_transaction(tr, log_ptr, object_name, "NRTM:ADD:While creating new"); */
	  /* For serials this is CREATE operation. Increase sequence to have it correct in serals */
          if(dummy==1) { tr->action=TA_CREATE; tr->sequence_id++; } 
	  /* create ADD serial record */
          UD_lock_serial(tr);
	  serial_id = UD_create_serial(tr); 
	  CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
          UD_commit_serial(tr);
	  UD_unlock_serial(tr);
	  /* Mark TR as clean */
	  TR_mark_clean(tr);
          /* log the transaction */
          result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD");

        }
      }
    }
    else { /* ADD ADD =>brand new object*/
      if(tr->object_id==0) {
/*      fprintf(stderr,"CREATE new\n");*/
        /* Create an object and update NHR */
        tr->action=(TA_CREATE | ta_upd_nhr);
        object_process(tr);
        /* create ADD serial */
	UD_lock_serial(tr);
	serial_id = UD_create_serial(tr); 
	CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
        UD_commit_serial(tr);
	UD_unlock_serial(tr);

	/* Mark TR as clean */
	TR_mark_clean(tr);
        /* log the transaction */
        result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD");
      }
      else { /* object already exists in the database */
	/* this may happen because of dummies*/
	/* or with some implementations of mirroring protocol that have atomic update */
	/* instead of add + del */
        tr->action=TA_UPDATE;
        dummy=isdummy(tr);
        /* If we are replacing dummy with a real object update NHR */
/*        if(dummy==1) tr->action = ( ta_upd_nhr| TA_UPD_DUMMY);*/
         if(dummy==1) tr->action = TA_UPD_DUMMY;
        object_process(tr);
        /* For serials this is CREATE operation. Increase sequence to have it correct in serals */
        if(dummy==1) { tr->action=TA_CREATE; tr->sequence_id++; }
	/* create ADD serial record */
	UD_lock_serial(tr);
	serial_id = UD_create_serial(tr); 
	CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
        UD_commit_serial(tr);
	UD_unlock_serial(tr);
	/* Mark TR as clean */
	TR_mark_clean(tr);
        /* log the transaction */
        result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD");
	
      } 
    }
    break;
    
  case OP_DEL:
    if(nrtm->tr){ /*DEL DEL =>saved */
      /* check this is not a deletion of the same object twise */
      /* this should not happen but we cannot trust foreign sources */
      /* in such case process saved transaction but fail the current one */
      if(nrtm->tr->object_id == tr->object_id) tr->object_id=0;
	    
      object_process(nrtm->tr); /* delete the previous(saved) object*/
      /* create DEL serial record */
      UD_lock_serial(nrtm->tr);
      serial_id = UD_create_serial(nrtm->tr);  
      CP_CREATE_S_PASSED(nrtm->tr->action); TR_update_status(nrtm->tr);
      UD_commit_serial(nrtm->tr);
      UD_unlock_serial(nrtm->tr);
      /* Mark TR as clean */
      TR_mark_clean(nrtm->tr);
      /* log the transaction */
      result=report_transaction(nrtm->tr, serial_id, log_ptr, &sotime, "M:DEL");
      
      object_free(nrtm->tr->object);
      transaction_free(nrtm->tr); nrtm->tr=NULL;
    }
    /* save the real object (not a dummy one ) */
    if(tr->object_id>0 && !isdummy(tr)){ 
      /* save the object*/
      tr->action=(TA_DELETE | ta_upd_nhr);
      nrtm->tr=tr;
      return(0);
    }
    else { /* this is an error - Trying to DEL non-existing object*/
      tr->succeeded=0; tr->error|=ERROR_U_COP;
      tr->action=(TA_DELETE | ta_upd_nhr);
      /* create and initialize TR record for crash recovery */
      TR_create_record(tr);
      /* create DEL serial record anyway */
      UD_lock_serial(tr);
      serial_id = UD_create_serial(tr); 
      CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
      UD_commit_serial(tr);
      UD_unlock_serial(tr);
      /* Mark TR as clean */
      TR_mark_clean(tr);
      /* log the transaction */
      result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:DEL: non-existing object");
      
    }
    break;
  
  default:
    tr->succeeded=0; tr->error |=ERROR_U_BADOP;
    break;  
  }

 /* Free resources */  
  object_free(tr->object);
  transaction_free(tr);
  
  return(result);
} /* process_nrtm() */



/************************************************************
* process_updates()                                         *
*                                                           *
* Process object in update mode                             *
*                                                           *
* ud_stream - pointer to UD_stream structure                *
* object_name - name of the object                          *
* operation - operation code (OP_ADD/OP_DEL)                *
*                                                           *
* Note:                                                     *
* Frees tr and tr->obj on exit                              *
*                                                           *
* Returns:                                                  *
* 0  - okay                                                 *
* <0- number of failed objects                              *
*                                                           * 
************************************************************/

static int process_updates(UD_stream_t *ud_stream, Transaction_t *tr, int operation)
{
int result=0;
Log_t *log_ptr= &(ud_stream->log);
int dummy=0;
ut_timer_t sotime;
int ta_upd_nhr;
char *reason;

    /* Start timer for statistics */
    UT_timeget(&sotime);
    if(IS_NO_NHR(tr->mode))ta_upd_nhr=0; else ta_upd_nhr = TA_UPD_NHR;

    switch(operation) {
    /* Compare operations and report an error if they do not match */    
    case OP_ADD:
      if(tr->object_id!=0) { /* trying to create, but object exists */
        tr->succeeded=0; tr->error|=ERROR_U_COP;
	reason="U:ADD:object already exists";
        g_string_sprintfa(tr->error_script,"E[%d]:NEW requested but object already exists\n" ,ERROR_U_COP);
        UD_ack(tr); /* Send a NACK */
      } else {
       /* Action: create the object and update NHR */
        tr->action=(TA_CREATE | ta_upd_nhr);
	reason="U:ADD";
        object_process(tr);
      }
      break;
    case OP_UPD:
      if(tr->object_id==0) { /* trying to update non-existing object*/
        tr->succeeded=0; tr->error|=ERROR_U_COP;
	reason="U:UPD:non-existing object";
        g_string_sprintfa(tr->error_script,"E[%d]:UPD requested but no existing object found\n" ,ERROR_U_COP);
        UD_ack(tr); /* Send a NACK */
      } else {
        tr->action=TA_UPDATE;
	reason="U:UPD";
        dummy=isdummy(tr);
        /* If we are replacing dummy with a real object update NHR */
/*        if(dummy==1) tr->action = (ta_upd_nhr | TA_UPD_DUMMY); */
        if(dummy==1) tr->action = TA_UPD_DUMMY;
        object_process(tr);
      }
      break;

    case OP_DEL:        
      if(tr->object_id==0) { /* trying t delete non-existing object*/
        tr->succeeded=0; tr->error|=ERROR_U_COP;
	reason="U:DEL:non-existing object";
        g_string_sprintfa(tr->error_script,"E[%d]:DEL requested but no existing object found\n" ,ERROR_U_COP);
	UD_ack(tr);
      } else {
        tr->action=(TA_DELETE | ta_upd_nhr);
	reason="U:DEL";
        object_process(tr);
      }
      break;
                
    default:                
      /* bad operation for this mode if not standalone */
      if(IS_STANDALONE(tr->mode)) {
        if(tr->object_id==0){
	  tr->action=(TA_CREATE | ta_upd_nhr); 
	  reason="U:ADD";
	}
	else {
	  tr->action=TA_UPDATE;
	  reason="U:UPD";
	}
        object_process(tr);
      }
      else {
        tr->succeeded=0; 
        tr->error|=ERROR_U_BADOP;
        g_string_sprintfa(tr->error_script,"E[%d]:Unknown operation requested\n" ,ERROR_U_BADOP);
	reason="U:bad operation";
        UD_ack(tr); /* Send a NACK */ 
      }
      break;
    }
   /* If not in standalone mode create serial and copy error transcript */ 
    if(!IS_STANDALONE(tr->mode)) {
      if(tr->succeeded){
	      /* we don't want to generate DEL serial for dummy replacement*/
	      if(dummy==1) { tr->action=TA_CREATE; tr->sequence_id++; }
              UD_lock_serial(tr);
	      UD_create_serial(tr); 
	      CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
	      UD_commit_serial(tr);
	      UD_unlock_serial(tr);
	      /* Mark the TR as clean */
              TR_mark_clean(tr);
      }
    }  
   
   /* Make a report. U stands for update stream. No reason */
    result=report_transaction(tr, tr->transaction_id, log_ptr, &sotime, reason);

   /* Free resources */   
    object_free(tr->object);
    transaction_free(tr);
    
    return(result);
        
} /* process_updates() */


/************************************************************
*                                                           *
* int process_transaction()                                 *
*                                                           *
* Processes the transaction                                 *
*                                                           *
* ud_stream - pointer to UD_stream_t structure              *
*                                                           *
* Returns:                                                  *
* 0 - no error                                              *
* <0- number of failed objects                              *
*                                                           *
************************************************************/

/* It frees the obj */

static int process_transaction(UD_stream_t *ud_stream, 
                         Obj_parse_t *parse,
			 int operation,
			long transaction_id)
{
Transaction_t *tr = NULL;
Object_t *obj = parse->obj;
int result;

/* check if the requested transaction has already been processed */
/* this may happen in case of crash. If so, just send an ack and return */
 if(TR_check(ud_stream->db_connection, transaction_id, (ud_stream->condat).sock))return(1);

/* start new transaction now */ 
 tr = transaction_new(ud_stream->db_connection, obj->type);

/* Return with error if transaction cannot be created */ 
 if (tr == NULL) die;
 
 tr->mode=ud_stream->ud_mode;
 tr->load_pass=ud_stream->load_pass;
 tr->object=obj;
 tr->nh=parse->nh_ptr;
 tr->source_hdl=ud_stream->source_hdl;
 tr->socket=(ud_stream->condat).sock;
 tr->transaction_id=transaction_id;
 
/* We perform no commit/rollback in the loader mode, so thread_id should be set to 0 */
 if(ud_stream->load_pass!=0) { tr->thread_ins=0; tr->thread_upd=0; }
    
/* For the first load pass we only create objects */ 
 if(ud_stream->load_pass==1) {
	 Object_t *obj=tr->object;
	 /* still we need to fill tr->K (last.pkey) field */ 
	 g_slist_foreach(obj->attributes, ud_each_primary_key_select, tr);
	 tr->object_id=0;
 }
 else tr->object_id=get_object_id(tr);
 
/* Object cannot be retrieved */
 if(tr->object_id==-1) { /* DB error*/
    tr->succeeded=0;
    tr->error |= ERROR_U_DBS;
    ER_perror(FAC_UD, UD_SQL, "%s: Object cannot be retrieved", parse->object_name);
    die;
    transaction_free(tr);
    object_free(obj);
 }
/* save the name of person/role as we need it for referential */
/* integrity check when deleting the object against names. */
/* This is needed to support legacy references by name rather */
/* then by nic_hdl */
  if((tr->class_type==C_PN) || (tr->class_type==C_RO)){
    /* Save the value */
    tr->save=g_strdup(parse->object_name);
/*    attribute_free(attr, NULL); */
  }
                                               
/* Process transaction. tr and obj are freed inside the process_* functions */

 if(IS_UPDATE(ud_stream->ud_mode))
 /* We are in update mode */
    result=process_updates(ud_stream, tr, operation);
 else
 /* We are in NRTM mode */   
    result=process_nrtm(ud_stream, tr, operation);

 return(result);

}          
          

/************************************************************
*                                                           *
* int UD_process_stream(UD_stream_t *ud_stream)             *
*                                                           *
* Processes the stream                                      *
*                                                           *
* ud_stream - pointer to UD_stream_t structure              *
*                                                           *
* Returns:                                                  *
* in update mode (!standalone)(1 object processed):         *
* 1 - no error                                              *
* <0- errors                                                *
*                                                           *
* in NRTM & standalone modes                                *
* total number of object processed                          *
*                                                           *
************************************************************/

int UD_process_stream(UD_stream_t *ud_stream)
{
  char line_buff[STR_XXL];
  Object_t *obj = NULL;
  SQ_connection_t *sql_connection;
  int start_object;
  int a_type;
  struct _nrtm *nrtm;
  Log_t *log_ptr= &(ud_stream->log);
  ut_timer_t stime, ftime, sotime;
  float obj_second1, obj_second10, timediff;
  int result=0;
  int operation=0;
  int interrupt=0;
  int do_update;
  int default_ud_mode = ud_stream->ud_mode;
  Line_Type_t linetype;
  Transaction_t *tr;
  long transaction_id=0; /* transaction_id (to be supplied by DBupdate and stored in Database) */
  long serial_id;
  Obj_parse_t obj_parse; /* the structure used to parse a text object */
  
  
  
  
  nrtm=ud_stream->nrtm;
  start_object = 1;
  a_type=-1; 


  /* Check connection to the database */
  if(SQ_ping(ud_stream->db_connection)) {
   ER_perror(FAC_UD, UD_SQL, "%s", SQ_error(ud_stream->db_connection));
   die;
  }
   
  sql_connection=ud_stream->db_connection;
  
  ud_parse_init(sql_connection, &obj_parse);
  /* Start timer for statistics */
  UT_timeget(&stime);

 /* Main loop. Reading input stream line by line */
 /* Empty line signals to start processing an object, if we have it */ 
  while (SK_cd_gets(&ud_stream->condat, line_buff, sizeof(line_buff))>0) {


    switch (linetype=line_type(line_buff, &transaction_id)) {
      case LINE_ATTRIBUTE:
       /* parse the object line by line */
       obj = UD_parse_object(&obj_parse, line_buff);

      break;

      case LINE_COMMENT:
      break;

      case LINE_EOF:
      break;

      case LINE_ACK:
       tr = transaction_new(ud_stream->db_connection, 0);
       tr->transaction_id=transaction_id;
       TR_delete_record(tr);
       transaction_free(tr);
      break;
      
      
      case LINE_ADD:
      /* restore the default operation mode */
       operation=OP_ADD;
       ud_stream->ud_mode=default_ud_mode;
      break;
      
      case LINE_OVERRIDE_ADD:
      /* for override - switch the dummy bit on */
       operation=OP_ADD;
       ud_stream->ud_mode=default_ud_mode|B_DUMMY;
      break;
      
      case LINE_UPD:
      /* restore the default operation mode */
       operation=OP_UPD;
       ud_stream->ud_mode=default_ud_mode;
      break;  

      case LINE_OVERRIDE_UPD:
      /* for override - switch the dummy bit on */
       operation=OP_UPD;
       ud_stream->ud_mode=default_ud_mode|B_DUMMY;
      break;
      
      case LINE_DEL:
      /* restore the default operation mode */
       operation=OP_DEL;
       ud_stream->ud_mode=default_ud_mode;
      break; 

      case LINE_OVERRIDE_DEL:
      /* for override - switch the dummy bit on */
       operation=OP_DEL;
       ud_stream->ud_mode=default_ud_mode|B_DUMMY;
      break;
 
      case LINE_EMPTY:
       /* start processing the object */
        if ((obj=obj_parse.obj)) { /* if not just garbage*/
	 /* normalize the object (reorder and split attributes */
	 ud_normalize(&obj_parse);
 	 /* XXX */
/*	 print_object(obj); */

         /* start new transaction now */
	 ER_dbg_va(FAC_UD, ASP_UD_OBJ, "[%ld] object: [%s] ", transaction_id, obj_parse.object_name); 

/*	 fprintf(stderr, "transction # %ld\n", transaction_id); */
         result=process_transaction(ud_stream, &obj_parse, operation, transaction_id);
         /* process_transaction() frees tr and obj structures, */
         /* so make sure we'll not reference these objects in the future */
         operation=OP_NOOP;
         transaction_id=0;
         ud_stream->ud_mode=default_ud_mode;
	 ud_parse_free(&obj_parse);
          
         /* this is a good place for quick interrupt */
         do_update=CO_get_do_update();
         if (do_update) interrupt=0; else interrupt=1;
	} /* if this is a real object */
	/* initialize the parsing structure */
	ud_parse_init(sql_connection, &obj_parse);

      break;

      default:
	die;
    } /* switch */
    
    /* Finish processing if interrupt has been set */
    if (interrupt) break;
  } /* Main loop of data stream processing : while */
 
 /* Some postprocessing */
  if(IS_NRTM_CLNT(ud_stream->ud_mode)){
  /* We are in NRTM mode */
  /* Clean up */
/*   fclose(ud_stream->stream); */
  /* In NRTM mode there may be a saved object that is unprocessed */   
   if(nrtm->tr){ /*saved backlog?*/
    /* restart the timer for statistics */
    UT_timeget(&sotime);
    object_process(nrtm->tr); /* delete the previous(saved) object*/
/*    result=report_transaction(nrtm->tr, &(ud_stream->log), nrtm->object_name, 
                              "NRTM:DEL:While deleting previous(saved) object"); */
    /* create DEL serial record no matter what the result is */
    UD_lock_serial(nrtm->tr);
    serial_id = UD_create_serial(nrtm->tr); 
    CP_CREATE_S_PASSED(nrtm->tr->action); TR_update_status(nrtm->tr);
    UD_commit_serial(nrtm->tr);
    UD_unlock_serial(nrtm->tr);
    /* Mark TR as clean */
    TR_mark_clean(nrtm->tr);
    /* log the transaction */
    result=report_transaction(nrtm->tr, serial_id, log_ptr, &sotime, "M:DEL");

    object_free(nrtm->tr->object);
    transaction_free(nrtm->tr); nrtm->tr=NULL;
   } 
  }

 /* That's all. Free GString */
/*  g_string_free(g_line_buff, TRUE);*/

                                                                                                       
 /* Calculate some statistics */
/*  ftime=time(NULL); */
  UT_timeget(&ftime);
/*  obj_second1 = (float)(log_ptr->num_ok)/(ftime-stime);
  obj_second10 = (float)(log_ptr->num_ok+log_ptr->num_failed)/(ftime-stime); */
  timediff = UT_timediff(&stime, &ftime);
  obj_second1 = (float)(log_ptr->num_ok)/timediff;
  obj_second10 = (float)(log_ptr->num_ok+log_ptr->num_failed)/timediff;
  
  /* Print the report */
  if(IS_STANDALONE(ud_stream->ud_mode) || (!IS_UPDATE(ud_stream->ud_mode))) {

   ER_inf_va(FAC_UD, ASP_UD_UPDLOG,"%s ******** report **********", UD_TAG);
   ER_inf_va(FAC_UD, ASP_UD_UPDLOG,"%s %d objects OK (%7.4f obj/s)", UD_TAG, log_ptr->num_ok, obj_second1);
   ER_inf_va(FAC_UD, ASP_UD_UPDLOG,"%s %d objects failed", UD_TAG, log_ptr->num_failed);
   ER_inf_va(FAC_UD, ASP_UD_UPDLOG,"%s average processing time %7.4f obj/s (%6.2f obj/min)", UD_TAG, 
                          obj_second10, obj_second10*60);
   result=log_ptr->num_ok+log_ptr->num_failed;
  }
  return(result);

} /* UD_process_stream */

