modules/ud/ud_process_stream.c
/* [<][>][^][v][top][bottom][index][help] */
FUNCTIONS
This source file includes following functions.
- get_NRTM_stream
- report_transaction
- s_split
- escape_apostrophes
- line_type
- process_nrtm
- process_updates
- UD_process_stream
/***************************************
$Revision: 1.7 $
Functions to process data stream( file, network socket, etc.)
Status: NOT REVUED, NOT TESTED
Author(s): Chris Ottrey, Andrei Robachevsky
******************/ /******************
Modification History:
andrei (17/01/2000) Created.
******************/ /******************
Copyright (c) 2000 RIPE NCC
All Rights Reserved
Permission to use, copy, modify, and distribute this software and its
documentation for any purpose and without fee is hereby granted,
provided that the above copyright notice appear in all copies and that
both that copyright notice and this permission notice appear in
supporting documentation, and that the name of the author not be
used in advertising or publicity pertaining to distribution of the
software without specific, written prior permission.
THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING
ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS; IN NO EVENT SHALL
AUTHOR BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY
DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
***************************************/
#include <sys/types.h>
#include <sys/socket.h>
#include <netdb.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <sys/stat.h>
#include <fcntl.h>
#include "ud.h"
#include "ud_int.h"
typedef enum _Line_Type_t {
LINE_ATTRIBUTE,
LINE_COMMENT,
LINE_EMPTY,
LINE_EOF,
LINE_ADD,
LINE_UPD,
LINE_DEL
} Line_Type_t;
/* Maximum number of objects(serials) we can consume at a time */
#define SBUNCH 1000
static int report_transaction(Transaction_t *tr, Log_t *log, char *obj_name, char *reason);
static char *s_split(char *line);
static int process_nrtm(Transaction_t *tr, struct _nrtm *nrtm, Log_t *log, char *object_name, int operation);
static int process_updates(Transaction_t *tr, Log_t *log, char *object_name, int operation, int standalone);
/* temporary files to download serials */
char tmpfile1[STR_S], tmpfile2[STR_S];
FILE *get_NRTM_stream(struct _nrtm *nrtm, int upto_last)
/* [<][>][^][v][top][bottom][index][help] */
{
int sockfd;
struct hostent *hptr;
struct sockaddr_in serv_addr;
struct in_addr *paddr;
char line_buff[STR_XXL];
FILE *fp;
int fd;
int fdtmp;
int nread, nwrite;
struct hostent result;
int error;
fprintf(stderr, "Making connection to NRTM server ...\n");
if ((sockfd=socket(AF_INET, SOCK_STREAM, 0))==-1){
perror("socket");
return(NULL);
}
// hptr=gethostbyname(nrtm->server);
hptr=gethostbyname_r(nrtm->server, &result, line_buff, sizeof(line_buff), &error);
if (hptr) { // this is a network stream
paddr=(struct in_addr *)hptr->h_addr;
bzero(&serv_addr, sizeof(serv_addr));
serv_addr.sin_family=AF_INET;
serv_addr.sin_port=nrtm->port;
memcpy(&serv_addr.sin_addr, paddr, sizeof(struct in_addr));
fprintf(stderr,"Trying %s port %d\n", inet_ntoa(serv_addr.sin_addr), nrtm->port);
if(connect(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr))==-1) {
perror("connect");
return(NULL);
}
fprintf(stderr, "Sending Invitation\n");
if(upto_last)
sprintf(line_buff, "-g RIPE:%d:%d-LAST\n", nrtm->version, nrtm->current_serial+1);
else
sprintf(line_buff, "-g RIPE:%d:%d-%d\n", nrtm->version, nrtm->current_serial+1, nrtm->current_serial+SBUNCH);
write(sockfd, line_buff, strlen(line_buff));
fd=sockfd;
fprintf(stderr, "%s", line_buff);
fprintf(stderr, "Returning stream pointer\n");
}
else { // this is a file stream
fprintf(stderr, "Trying file ...\n");
if((fd=open(nrtm->server, O_RDONLY, 0666))==-1) {
perror("open");
return(NULL);
}
}
/* make temporary files */
sprintf(tmpfile1, "temp.XXXX");
if((fdtmp=mkstemp(tmpfile1))==-1) {
perror("mkstemp");
close(fd);
return(NULL);
}
while ((nread=read(fd, line_buff, sizeof(line_buff)))) {
if(nread==-1) return(NULL);
nwrite=write(fdtmp, line_buff,nread);
if(nread != nwrite) return(NULL);
}
close(fd); close(fdtmp);
sprintf(tmpfile2, "%s.2", tmpfile1);
sprintf(line_buff, "cat %s | ./ripe2rpsl > %s", tmpfile1, tmpfile2);
if (system(line_buff)!=0) return(NULL);
if((fp=fopen(tmpfile2, "r+"))==NULL)return(NULL);
unlink(tmpfile1);
return(fp);
}
/******************************************************************
* report_transaction() *
* *
* Prints error report to the log *
* *
* reason - additional message thet will be included *
* *
* *****************************************************************/
static int report_transaction(Transaction_t *tr, Log_t *log, char *obj_name, char *reason)
/* [<][>][^][v][top][bottom][index][help] */
{
int result=0;
if(tr->succeeded==0) {
result=tr->error;
log->num_failed++;
printf("FAILED[%s][%s(%d)](%d/%d)\n ", obj_name, reason, result, log->num_failed, (log->num_failed)+(log->num_ok));
fprintf(log->logfile, "*FAILED[%s][%s](%d/%d)\n ", obj_name, reason, log->num_failed, (log->num_failed)+(log->num_ok));
if(result & ERROR_U_MEM) fprintf(log->logfile, "\t*Memory allocation error\n");
if(result & ERROR_U_DBS) fprintf(log->logfile, "\t*Database (SQL) error\n");
if(result & ERROR_U_OBJ) fprintf(log->logfile, "\t*Object (RF) error\n");
if(result & ERROR_U_AUT) fprintf(log->logfile, "\t*Object authentication error\n");
if(result & ERROR_U_BADOP) fprintf(log->logfile, "\t*Bad operation\n");
if(result & ERROR_U_COP) fprintf(log->logfile, "\t*Conflicting operation\n");
if(result & ERROR_U_NSUP) fprintf(log->logfile, "\t*Object of this type is not supported\n");
if(result & ERROR_U_BUG) fprintf(log->logfile, "\t*Software bug - report to <ripe-dbm@ripe.net>\n");
fprintf(log->logfile, "%s", (tr->error_script)->str);
result=(-1)*result;
fflush(log->logfile);
}
else {
result=1;
log->num_ok++;
printf("OK(%d/%d)\n", log->num_ok, (log->num_failed)+(log->num_ok));
}
return(result);
}
/******************************************************************
* char *s_split(char *line) *
* consequently returns words (separated by whitespace in the line)*
* NULL - end. You need to retreive all words ! *
* *
* *****************************************************************/
static char *s_split(char *line)
/* [<][>][^][v][top][bottom][index][help] */
{
static char *delim;
static char *token=NULL;
if(token==NULL)token=line;
else token=delim;
if(token==NULL)return(token);
while(isspace((int)*token))token++;
delim=token;
while(!isspace((int)*delim)) {
if((*delim)=='\0'){
if(delim==token)token=NULL;
delim=NULL; return(token);
}
delim++;
}
*delim='\0'; delim++;
return(token);
}
GString *escape_apostrophes(GString *text) {
/* [<][>][^][v][top][bottom][index][help] */
int i;
for (i=0; i < text->len; i++) {
if ((text->str[i] == '\'') || (text->str[i] == '\\')) {
text = g_string_insert_c(text, i, '\\');
i++;
}
}
return(text);
} /* escape_apostrophes() */
static Line_Type_t line_type(const char *line) {
/* [<][>][^][v][top][bottom][index][help] */
Line_Type_t result = -1;
if (strncmp(line, "# EOF", 4) == 0) {
result = LINE_EOF;
}
else if (strncmp(line, "#", 1) == 0) {
result = LINE_COMMENT;
}
else if (strcmp(line, "\n") == 0) {
result = LINE_EMPTY;
}
else if (strncmp(line, "ADD", 3) == 0) {
result = LINE_ADD;
}
else if (strncmp(line, "UPD", 3) == 0) {
result = LINE_UPD;
}
else if (strncmp(line, "DEL", 3) == 0) {
result = LINE_DEL;
}
else {
result = LINE_ATTRIBUTE;
}
return result;
} /* line_type() */
/************************************************************
* process_nrtm() *
* *
* Process object in NRTM client mode *
* *
* nrtm - pointer to _nrtm structure *
* log - pointer to Log_t structure *
* object_name - name of the object *
* operation - operation code (OP_ADD/OP_DEL) *
* *
* Returns: *
* 1 - okay *
* <0 - error *
* *
************************************************************/
static int process_nrtm(Transaction_t *tr, struct _nrtm *nrtm, Log_t *log, char *object_name, int operation)
/* [<][>][^][v][top][bottom][index][help] */
{
int error=0;
//fprintf(stderr,"NRTM mode\n");
switch (operation) {
case OP_ADD:
if(nrtm->tr){ //saved?
if(tr->object_id==0) {
// fprintf(stderr,"DEL previous\n");
object_process(nrtm->tr); // delete the previous(saved) object
error=report_transaction(nrtm->tr, log, nrtm->object_name, "NRTM:DEL:While deleting previous(saved)");
object_free(nrtm->tr->object);
transaction_free(nrtm->tr); nrtm->tr=NULL;
tr->action=TR_CREATE;
// fprintf(stderr,"CREATE next\n");
object_process(tr); // create a new one
error=report_transaction(tr, log, object_name, "NRTM:ADD:While creating new");
}
else { //compare the two, may be we may collapse operations
if(tr->object_id==nrtm->tr->object_id) {
object_free(nrtm->tr->object);
transaction_free(nrtm->tr); nrtm->tr=NULL;
// fprintf(stderr,"DEL-ADD ->> UPDATE\n");
tr->action=TR_UPDATE;
object_process(tr);
report_transaction(tr, log, object_name,"NRTM:upd");
error=report_transaction(tr, log, object_name,"NRTM:upd");
}
else { // this should be a dummy object in the database or an interleaved operation
// fprintf(stderr,"DEL previous\n");
object_process(nrtm->tr); // delete the previous(saved) object
error=report_transaction(nrtm->tr, log, nrtm->object_name, "NRTM:DEL:While deleting previous(saved)");
object_free(nrtm->tr->object);
transaction_free(nrtm->tr); nrtm->tr=NULL;
tr->action=TR_UPDATE;
// fprintf(stderr,"UPDATE next(dummy)\n");
object_process(tr); // create a new one
error=report_transaction(tr, log, object_name, "NRTM:ADD:While creating new");
}
}
}
else { // brand new object
if(tr->object_id==0) {
// fprintf(stderr,"CREATE new\n");
tr->action=TR_CREATE;
object_process(tr);
error=report_transaction(tr, log, object_name,"NRTM:ADD:While creating new");
}
else { // this may happen because of dummies
// fprintf(stderr,"CREATE new\n");
tr->action=TR_UPDATE;
object_process(tr);
error=report_transaction(tr, log, object_name,"NRTM:ADD:While creating new");
}
}
break;
case OP_DEL:
if(nrtm->tr){ //saved?
// fprintf(stderr,"DEL previous\n");
object_process(nrtm->tr); // delete the previous(saved) object
error=report_transaction(nrtm->tr, log, nrtm->object_name, "NRTM:DEL:While deleting previous(saved) object");
object_free(nrtm->tr->object);
transaction_free(nrtm->tr); nrtm->tr=NULL;
}
if(tr->object_id>0){ // save the object
// fprintf(stderr,"SAVE del object\n");
tr->action=TR_DELETE;
nrtm->tr=tr;
strcpy(nrtm->object_name, object_name);
return(error);
}
else { // this is an error
tr->succeeded=0; tr->error|=ERROR_U_COP;
error=report_transaction(tr, log, object_name, "NRTM:OOS:Trying to DEL non-existing object");
// fprintf(stderr,"Lost sync. Skipping\n");
}
break;
default:
tr->succeeded=0; tr->error |=ERROR_U_BADOP;
break;
}
object_free(tr->object);
transaction_free(tr);
return(error);
} /* process_nrtm() */
/************************************************************
* process_updates() *
* *
* Process object in update mode *
* *
* nrtm - pointer to _nrtm structure *
* log - pointer to Log_t structure *
* object_name - name of the object *
* operation - operation code (OP_ADD/OP_DEL) *
* standalone - if the program is executed standalone *
* *
* Returns: *
* 1 - okay *
* <0 - error *
* *
************************************************************/
static int process_updates(Transaction_t *tr, Log_t *log, char *object_name, int operation, int standalone)
/* [<][>][^][v][top][bottom][index][help] */
{
int error=0;
switch(operation) {
case OP_ADD:
case OP_UPD:
if(tr->object_id==0) tr->action=TR_CREATE; else tr->action=TR_UPDATE;
object_process(tr);
break;
case OP_DEL:
if(tr->object_id==0) { // trying t delete non-existing object
tr->succeeded=0; tr->error|=ERROR_U_COP;
} else {
tr->action=TR_DELETE;
object_process(tr);
}
break;
default:
/* bad operation for this mode if not standalone */
if(standalone) {
if(tr->object_id==0)tr->action=TR_CREATE; else tr->action=TR_UPDATE;
object_process(tr);
}
else {
tr->succeeded=0;
tr->error|=ERROR_U_BADOP;
}
break;
}
error=report_transaction(tr, log, object_name, "UPD");
return(error);
} /* process_updates() */
/************************************************************
* *
* int UD_process_stream(UD_stream_t *ud_stream) *
* *
* Processes the stream *
* *
* ud_stream - pointer to UD_stream_t structure *
* *
* Returns: *
* in update mode (!standalone)(1 object processed): *
* 1 - no error *
* <0- errors *
* *
* in NRTM & standalone modes *
* total number of object processed *
* *
************************************************************/
int UD_process_stream(UD_stream_t *ud_stream)
/* [<][>][^][v][top][bottom][index][help] */
{
FILE *logfile;
char line_buff[STR_XXL], object_name[STR_XXL];
GString *g_line_buff; // needed to escape apostrophes
Attribute_t *attr, *attr_split;
Attribute_t *mnt_by; // we need this for reordering mnt_by and member_of (member_of should come after)
Attribute_t *nic_hdl; // we need this for reordering nic_hdl and admin_c, etc. (admin_c should come after)
Object_t *obj = NULL;
Transaction_t *tr = NULL;
SQ_connection_t *sql_connection;
int start_object;
int a_type;
char *s_attr;
long num_skip;
struct _nrtm *nrtm;
Log_t log;
time_t stime, ftime;
double obj_second1, obj_second10;
int standalone;
int error;
int operation=0;
nrtm=ud_stream->nrtm;
standalone=IS_STANDALONE(ud_stream->ud_mode);
start_object = 1;
if ((g_line_buff = g_string_sized_new(STR_XXL)) == NULL){ fprintf(stderr, "E: cannot allocate gstring\n"); return(-1); }
fprintf(stderr, "D: Making SQL connection to %s@%s ...", ud_stream->db_name, ud_stream->db_host);
sql_connection = SQ_get_connection(ud_stream->db_host, ud_stream->db_port,
ud_stream->db_name, ud_stream->db_user, ud_stream->db_pswd);
if(!sql_connection) {
fprintf(stderr, "D: ERROR: no SQL connection\n");
return(-1);
}
fprintf(stderr, "OK\n");
logfile = fopen(ud_stream->log, "a+");
log.logfile=logfile;
log.num_ok=0; log.num_failed=0;
/* This is useful for loading DB from huge disk file. */
num_skip=ud_stream->num_skip;
if(num_skip>0) fprintf(stderr, "skipping %lu records\n", num_skip);
stime=time(NULL);
while (fgets(line_buff, STR_XXL, ud_stream->stream) != NULL) {
switch (line_type(line_buff)) {
case LINE_ATTRIBUTE:
if (start_object == 1) {
if(num_skip>0){ printf("\r%10lu", num_skip); num_skip--; log.num_ok++; break; }
mnt_by=NULL;
nic_hdl=NULL;
strncpy(object_name, line_buff, strlen(line_buff)-1);
*(object_name+strlen(line_buff)-1)='\0';
obj = object_new(line_buff);
if (obj) {
start_object = 0;
printf("D: object: [%s] ", object_name);
}
}
if (obj != NULL) {
g_string_sprintf(g_line_buff, "%s", line_buff);
g_line_buff=escape_apostrophes(g_line_buff);
attr = attribute_new(g_line_buff->str);
g_string_sprintfa(obj->object, "%s", g_line_buff->str);
if (attr != NULL) {
switch (a_type=(attr->type)) {
case A_MB: mnt_by=attr;
break;
case A_NH: nic_hdl=attr;
break;
case A_PN:
case A_RO:
case A_MR:
case A_SD:
case A_RZ:
case A_NS: //these attributes may appear several on the line - split them
while((s_attr=s_split(attr->value))){
attr_split = attribute_new1(a_type, s_attr);
obj->attributes = g_slist_append(obj->attributes, attr_split);
}
attribute_free(attr, NULL);
attr=NULL;
break;
default: break;
}
if(attr){ obj->attributes = g_slist_append(obj->attributes, attr);
}
}
}
break;
case LINE_COMMENT:
break;
case LINE_EOF:
break;
case LINE_ADD:
operation=OP_ADD;
break;
case LINE_UPD:
operation=OP_UPD;
break;
case LINE_DEL:
operation=OP_DEL;
break;
case LINE_EMPTY:
start_object=1;
if (obj != NULL) {
/* reorder some attributes */
if(mnt_by){
obj->attributes = g_slist_remove(obj->attributes, mnt_by);
obj->attributes = g_slist_insert(obj->attributes, mnt_by, 1);
}
if(nic_hdl){
obj->attributes = g_slist_remove(obj->attributes, nic_hdl);
obj->attributes = g_slist_insert(obj->attributes, nic_hdl, 1);
}
tr = transaction_new(sql_connection, obj->type);
if (tr != NULL) {
tr->standalone=standalone;
tr->dummy=IS_DUMMY_ALLOWED(ud_stream->ud_mode);
tr->load_pass=ud_stream->load_pass;
tr->object=obj;
if(ud_stream->load_pass) { tr->thread_ins=0; tr->thread_upd=0; }
if(ud_stream->load_pass==1) tr->object_id=0;
else tr->object_id=get_object_id(tr);
if(tr->object_id==-1) { // some error
tr->succeeded=0;
report_transaction(tr, &log, object_name, "DB error");
transaction_free(tr); tr=NULL;
object_free(obj); obj=NULL;
operation=OP_NOOP;
break;
}
else
if(nrtm) { //NRTM mode
error=process_nrtm(tr, nrtm, &log, object_name, operation);
tr=NULL;
obj=NULL;
operation=OP_NOOP;
}
else { //update mode
error=process_updates(tr, &log, object_name, operation, standalone);
if(!standalone) { /* copy the script */
ud_stream->error_script=g_strdup((tr->error_script)->str);
object_free(tr->object); obj=NULL;
transaction_free(tr); tr=NULL;
g_string_free(g_line_buff, TRUE);
SQ_close_connection(sql_connection);
fclose(logfile);
return(error);
}
else {
object_free(tr->object); obj=NULL;
transaction_free(tr); tr=NULL;
operation=OP_NOOP;
}
}
/* this is a good place for quick inerrupt */
}
}
break;
default:
fprintf(stderr, "ERROR: Bad line type\n");
} /* switch */
} /* while */
if(nrtm)
if(nrtm->tr){ //saved backlog?
object_process(nrtm->tr); // delete the previous(saved) object
report_transaction(nrtm->tr, &log, nrtm->object_name, "NRTM:DEL:While deleting previous(saved) object");
object_free(nrtm->tr->object);
transaction_free(nrtm->tr); nrtm->tr=NULL;
}
ftime=time(NULL);
obj_second1 = (float)(log.num_ok)/(ftime-stime);
obj_second10 = (float)(log.num_ok+log.num_failed)/(ftime-stime);
SQ_close_connection(sql_connection);
g_string_free(g_line_buff, TRUE);
fclose(ud_stream->stream);
if(IS_STANDALONE(ud_stream->ud_mode) || (!IS_UPDATE(ud_stream->ud_mode)))unlink(tmpfile2);
printf("\n\n******** report **********\n%d objects OK\n%d objects failed\n", log.num_ok, log.num_failed);
fprintf(logfile,"\n******** report **********\n");
fprintf(logfile," %d objects OK (%5.2f obj/s)\n", log.num_ok, obj_second1);
fprintf(logfile," %d objects failed\n", log.num_failed);
fprintf(logfile," average processing time %5.2f obj/s (%5.2f obj/min)\n", obj_second10, obj_second10*60);
fclose(logfile);
return(log.num_ok+log.num_failed);
} /* UD_process_stream */