1 | /***************************************
2 | $Revision: 1.50 $
3 |
4 | Functions to process data stream( file, network socket, etc.)
5 |
6 | Status: NOT REVUED, NOT TESTED
7 |
8 | Author(s): Chris Ottrey, Andrei Robachevsky
9 |
10 | ******************/ /******************
11 | Modification History:
12 | andrei (17/01/2000) Created.
13 | ******************/ /******************
14 | Copyright (c) 2000 RIPE NCC
15 |
16 | All Rights Reserved
17 |
18 | Permission to use, copy, modify, and distribute this software and its
19 | documentation for any purpose and without fee is hereby granted,
20 | provided that the above copyright notice appear in all copies and that
21 | both that copyright notice and this permission notice appear in
22 | supporting documentation, and that the name of the author not be
23 | used in advertising or publicity pertaining to distribution of the
24 | software without specific, written prior permission.
25 |
26 | THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING
27 | ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS; IN NO EVENT SHALL
28 | AUTHOR BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY
29 | DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
30 | AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
31 | OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
32 | ***************************************/
33 | #include <sys/types.h>
34 | #include <sys/socket.h>
35 | #include <netdb.h>
36 | #include <arpa/inet.h>
37 | #include <unistd.h>
38 | #include <sys/stat.h>
39 | #include <fcntl.h>
40 | #include <string.h>
41 | #include "constants.h"
42 | #include "query_command.h"
43 | #include "ud.h"
44 | #include "ud_int.h"
45 | #include "ud_tr.h"
46 | #include "timediff.h"
47 |
48 | typedef enum _Line_Type_t {
49 | LINE_ATTRIBUTE,
50 | LINE_COMMENT,
51 | LINE_EMPTY,
52 | LINE_EOF,
53 | LINE_ADD,
54 | LINE_UPD,
55 | LINE_DEL,
56 | LINE_OVERRIDE_ADD,
57 | LINE_OVERRIDE_UPD,
58 | LINE_OVERRIDE_DEL,
59 | LINE_ACK
60 | } Line_Type_t;
61 |
62 | /* Maximum number of objects(serials) we can consume at a time */
63 | #define SBUNCH 1000
64 |
65 | static int report_transaction(Transaction_t *tr, long transaction_id, Log_t *log, ut_timer_t *psotime, char *reason);
66 | static int process_nrtm(UD_stream_t *ud_stream, Transaction_t *tr, int operation);
67 | static int process_updates(UD_stream_t *ud_stream, Transaction_t *tr, int operation);
68 | static int process_transaction(UD_stream_t *ud_stream, Obj_parse_t *parse, int operation, long transaction_id);
69 |
70 | static GSList *ud_split_attribute(GSList *attr_list, Attribute_t *attr);
71 | static void ud_parse_init(SQ_connection_t *sql_connection, Obj_parse_t *parse);
72 | static void ud_parse_free(Obj_parse_t *parse);
73 | static int line_continuation(char *line);
74 | static GString *escape_apostrophes(GString *text);
75 | static Object_t *UD_parse_object(Obj_parse_t *parse, char *line_buff);
76 | static int ud_replace_substring(GString *gstring, char *old, char *new);
77 | static int ud_normalize(Obj_parse_t *parse);
78 |
79 | /* Delimiters that separate list members, both RPS(,) and legacy( ) */
80 | #define ATTR_DELIMITERS " ,"
81 |
82 |
83 | static void ud_parse_init(SQ_connection_t *sql_connection, Obj_parse_t *parse){
84 | bzero(parse, sizeof(Obj_parse_t));
85 | parse->garbage = 0;
86 | parse->sql_connection = sql_connection;
87 | }
88 |
89 | static void ud_parse_free(Obj_parse_t *parse){
90 | UT_free(parse->nic);
91 | UT_free(parse->object_name);
92 | }
93 |
94 |
95 | /******************************************************************
96 | * int line_continuation() *
97 | * *
98 | * Checks if the line starts with one of line continuation signs *
99 | * Returns 1 if this is line continuation, 0 otherwise *
100 | * *
101 | * ****************************************************************/
102 | static int line_continuation(char *line)
103 | {
104 | switch(*line) {
105 | case ' ':
106 | case '\t':
107 | case '+':
108 | return(1); /* these indicate line continuation */
109 | default: return(0);
110 | }
111 |
112 | }
113 |
114 | /******************************************************************
115 | * int ud_replace_substring() *
116 | * *
117 | * replace a subsring in a string *
118 | * Returns -1 if no match was found *
119 | * *
120 | * ****************************************************************/
121 | static int ud_replace_substring(GString *gstring, char *old, char *new)
122 | {
123 | int i, i0=-1, j=0;
124 |
125 | for (i=0; ((i < gstring->len) && (old[j] != '\0')); i++){
126 | if(gstring->str[i]==old[j]){
127 | if(i0==-1)i0=i;
128 | j++;
129 | } else {
130 | i0=-1;
131 | j=0;
132 | }
133 | }
134 | /* if we haven't reached the end of the old substring */
135 | /* or no match occured return -1 */
136 | if((old[j] != '\0') || (i0==-1)) return (-1);
137 | g_string_erase(gstring, i0, j);
138 | g_string_insert(gstring, i0, new);
139 | return(0);
140 | }
141 |
142 | /************************************************************
143 | * *
144 | * The function to splits attribute value into multiple *
145 | * words and appends them as attr_type - attr_valu pairs *
146 | * to the attr_list *
147 | * *
148 | * *
149 | ************************************************************/
150 | static GSList *ud_split_attribute(GSList *attr_list, Attribute_t *attr)
151 | {
152 | char *token;
153 | char *split;
154 | char *value, *n;
155 | Attribute_t *attr_split;
156 | GSList *the_list = attr_list;
157 |
158 |
159 | /* check for line continuation (+) */
160 | if (strncmp(attr->value, "+", 1) == 0) attr->value++;
161 | /* check for end-of-line comments */
162 | n = index(attr->value, '#');
163 | /* if there is no comment check for trailing \n */
164 | if(n == NULL) n = index(attr->value, '\n');
165 | /* now copy the clean value into the attribute */
166 | if(n == NULL) value = g_strdup(attr->value);
167 | else value = g_strndup(attr->value, (n - attr->value));
168 |
169 | token=value;
170 | while((split=strsep(&token, ATTR_DELIMITERS))){
171 | attr_split = attribute_new1(attr->type, split);
172 | if (attr_split) the_list = g_slist_append(the_list, attr_split);
173 | }
174 | free(value);
175 | attribute_free(attr, NULL);
176 |
177 | return(the_list);
178 | }
179 |
180 | /* XXX */
181 | static void each_attribute_print(void *element_data, void *tr_ptr)
182 | {
183 |
184 | Attribute_t *attr = (Attribute_t *)element_data;
185 |
186 | fprintf(stderr, "[%d|%s]\n", attr->type, attr->value);
187 |
188 | }
189 |
190 | /* XXX */
191 | static void print_object(Object_t *obj)
192 | {
193 | g_slist_foreach(obj->attributes, each_attribute_print, NULL);
194 | fprintf(stderr, ">>>>>\n%s\n", obj->object->str);
195 | }
196 |
197 |
198 | /******************************************************************
199 | * GString *escape_apostrophes() *
200 | * Escapes apostrophes in the text so they do not confuse printf *
201 | * functions and don't corrupt SQL queries *
202 | * *
203 | * *****************************************************************/
204 | static GString *escape_apostrophes(GString *text) {
205 | int i;
206 | for (i=0; i < text->len; i++) {
207 | if ((text->str[i] == '\'') || (text->str[i] == '\\')) {
208 | text = g_string_insert_c(text, i, '\\');
209 | i++;
210 | }
211 | }
212 | return(text);
213 | } /* escape_apostrophes() */
214 |
215 |
216 | /******************************************************************
217 | * Line_Type_t line_type(e) *
218 | * Determines the line type analysing the first letters *
219 | * *
220 | * ****************************************************************/
221 | static Line_Type_t line_type(const char *line, long *transaction_id) {
222 |
223 | if (strncmp(line, "# EOF", 4) == 0) return(LINE_EOF);
224 | if (strncmp(line, "#", 1) == 0) return(LINE_COMMENT);
225 | if (strcmp(line, "\n") == 0) return(LINE_EMPTY);
226 |
227 | if (strncmp(line, "ACK", 3) == 0) {
228 | *transaction_id = atol(line+3);
229 | return(LINE_ACK);
230 | }
231 | if (strncmp(line, "ADD_OVERRIDE", 12) == 0) {
232 | *transaction_id = atol(line+12);
233 | return(LINE_OVERRIDE_ADD);
234 | }
235 | if (strncmp(line, "UPD_OVERRIDE", 12) == 0) {
236 | *transaction_id = atol(line+12);
237 | return(LINE_OVERRIDE_UPD);
238 | }
239 | if (strncmp(line, "DEL_OVERRIDE", 12) == 0) {
240 | *transaction_id = atol(line+12);
241 | return(LINE_OVERRIDE_DEL);
242 | }
243 |
244 | if (strncmp(line, "ADD", 3) == 0) {
245 | *transaction_id = atol(line+3);
246 | return(LINE_ADD);
247 | }
248 | if (strncmp(line, "UPD", 3) == 0) {
249 | *transaction_id = atol(line+3);
250 | return(LINE_UPD);
251 | }
252 | if (strncmp(line, "DEL", 3) == 0) {
253 | *transaction_id = atol(line+3);
254 | return(LINE_DEL);
255 | }
256 |
257 | /* Otherwise this is an attribute */
258 | return(LINE_ATTRIBUTE);
259 |
260 | } /* line_type() */
261 |
262 | /******************************************************************
263 | * Object_t *UD_parse_object() *
264 | * *
265 | * Parses the object accepting line by line *
266 | * Stores the object and list of attributes in Obj_parse_t struct *
267 | * *
268 | * ****************************************************************/
269 | static Object_t *UD_parse_object(Obj_parse_t *parse, char *line_buff)
270 | {
271 | GString *g_line_buff;
272 | Attribute_t *attr;
273 |
274 | /* in case of garbage */
275 | if(parse->garbage) return(NULL);
276 |
277 | if(parse->obj==NULL) {
278 | /* which means this is the start - create a new object*/
279 | parse->obj = object_new(line_buff);
280 | /* if this is not a class attribute - this is a garbage till the next blank line */
281 | if(parse->obj==NULL) {
282 | parse->garbage = 1;
283 | return(NULL);
284 | }
285 | }
286 |
287 | /* allocate buffer or the string */
288 | if ((g_line_buff = g_string_sized_new(STR_XXL)) == NULL){
289 | ER_perror(FAC_UD, UD_MEM, "cannot allocate gstring");
290 | die;
291 | }
292 | /* copy the input */
293 | g_string_sprintf(g_line_buff, "%s", line_buff);
294 | /* escape apostrophes in the input line */
295 | g_line_buff=escape_apostrophes(g_line_buff);
296 |
297 | attr = attribute_new(g_line_buff->str);
298 | if (attr){
299 | /* save this attribute for future line continuations */
300 | /* skip attributes that are not stored in SQL database */
301 | if(DF_get_update_query_type(attr->type)==UD_NULL_){
302 | parse->current_attr = NULL;
303 | attribute_free(attr, NULL);
304 | }
305 | else {
306 | parse->current_attr = attr;
307 | if((attr->type == A_MB) || (attr->type == A_NH)) {
308 | /* insert it at the beginning */
309 | /* mnt-by should go before member-of to allow correct membership autorization (still done in RIPupd) */
310 | /* nic-hdl should go before any admin-c, tech-c to prevent errrors in self referencing role objects */
311 | parse->obj->attributes = g_slist_insert(parse->obj->attributes, parse->current_attr, 1);
312 | } else {
313 | /* append it to the attribute list */
314 | parse->obj->attributes = g_slist_append(parse->obj->attributes, parse->current_attr);
315 | }
316 | }
317 | }
318 | /* if this is not a valid attribute, then this may be unknown attr or line continuation */
319 | else
320 | if(line_continuation(g_line_buff->str))
321 | {
322 | /* if parse->attr == NULL, then this attr is not stored in SQL database - skip it */
323 | if(parse->current_attr){
324 | /* this is continuation of a previous attribute */
325 | char *line_value, *new_value;
326 | char *n;
327 | /* normalize and append to current_attr->value */
328 | /* check for end-of-line comments */
329 | n = index(g_line_buff->str, '#');
330 | /* if there is no comment check for trailing \n */
331 | if(n == NULL) n = index(g_line_buff->str, '\n');
332 | /* now copy the clean value into the attribute */
333 | if(n == NULL) line_value = g_strdup(g_line_buff->str);
334 | else line_value = g_strndup(g_line_buff->str, (n - g_line_buff->str));
335 | /* replace line continuation character with ' ' */
336 | *line_value = ' ';
337 | new_value = g_strconcat(parse->current_attr->value, line_value, NULL);
338 | parse->current_attr = attribute_upd(parse->current_attr, parse->current_attr->type, new_value);
339 | UT_free(new_value);
340 | UT_free(line_value);
341 | }
342 | }
343 | /* otherwise, this is unknown attribute, so we need to break the continuation */
344 | else parse->current_attr=NULL;
345 | /* copy the line into object no matter whether it is an attribute or not (continualtion, etc.) */
346 | g_string_sprintfa(parse->obj->object, "%s", g_line_buff->str);
347 | g_string_free(g_line_buff, TRUE);
348 | return(parse->obj);
349 | }
350 |
351 | /******************************************************************
352 | * int each_attribute_2_pass() *
353 | * *
354 | * Assigns nic-handles in case of AUTO *
355 | * Splits attributes *
356 | * * *
357 | ******************************************************************/
358 | void each_attribute_2_pass(void *element_data, void *ptr)
359 | {
360 | Attribute_t *attr = (Attribute_t *)element_data;
361 | Obj_parse_t *parse = (Obj_parse_t *)ptr;
362 |
363 |
364 | /* Strip the white space */
365 | g_strstrip(attr->value);
366 |
367 | switch(attr->type){
368 | case A_NH: /* nic-hdl */
369 | /* Parse the string into nh structure */
370 | /* In case of an AUTO NIC handle check the ID in the database */
371 | /* Possible errors leave to core processing */
372 | if(NH_parse(attr->value, &parse->nh_ptr) == 0) {
373 | /* this is an AUTO nic handle */
374 | /* ER_dbg_va(FAC_UD, ASP_UD_OBJ, "%s parsing nic handle: [%s]", UD_TAG, attr->value);*/
375 | /* Check if we can allocate it */
376 | if(NH_check(parse->nh_ptr, parse->sql_connection)>0){
377 | /* Convert nh to the database format */
378 | parse->nic = NH_convert(parse->nh_ptr);
379 | /* ER_dbg_va(FAC_UD, ASP_UD_OBJ, "%s parsed and converted nic handle: [%s]", UD_TAG, nic); */
380 | /* replace the auto nic with a real one */
381 | if(ud_replace_substring(parse->obj->object, attr->value, parse->nic)==-1) {
382 | ER_perror(FAC_UD, UD_BUG, "cannot replace auto nic handle");
383 | die;
384 | }
385 | /* Update the attribute */
386 | attr = attribute_upd(attr, attr->type, parse->nic);
387 |
388 | }
389 | }
390 | parse->new_attr_list = g_slist_append(parse->new_attr_list, attr);
391 | break;
392 | case A_PN: /* person */
393 | case A_RO: /* role */
394 | case A_MR: /* mbrs-by-ref */
395 | case A_MB: /* mnt-by */
396 | case A_ML: /* mnt-lower */
397 | case A_MO: /* member-of */
398 | case A_SD: /* sub-dom */
399 | case A_RZ: /* rev-srv */
400 | case A_NS: /* nserver */
401 | parse->new_attr_list = ud_split_attribute(parse->new_attr_list, attr);
402 | break;
403 | default:
404 | parse->new_attr_list = g_slist_append(parse->new_attr_list, attr);
405 | break;
406 | }
407 |
408 | }
409 |
410 |
411 | /******************************************************************
412 | * int ud_normalize() *
413 | * *
414 | * function accepts Obj_parse_t structure with attribute list *
415 | * *
416 | * It makes another pass to normalize object *
417 | * Normalization includes: *
418 | * *
419 | * 1) reorder attributes *
420 | * . mnt-by should go before member-of to allow correct *
421 | * membership autorization (still done in RIPupd) *
422 | * . nic-hdl should go before any admin-c, tech-c to prevent *
423 | * errrors in self referencing role objects *
424 | * 2) in case of pn/ro check nic handle and replace it in *
425 | * the obj->object if needed (AUTO) *
426 | * 3) split some attributes that allow lists *
427 | ******************************************************************/
428 | static int ud_normalize(Obj_parse_t *parse)
429 | {
430 | GSList *old_attr_list;
431 | GSList *first_element;
432 | Attribute_t *class_attr;
433 |
434 | old_attr_list = parse->obj->attributes;
435 | /* get class attribute - the first one */
436 | first_element = g_slist_nth(old_attr_list, 0);
437 | class_attr = (Attribute_t *)first_element->data;
438 | /* save object name for reporting and checking ref integrity for names (legacy) */
439 | parse->object_name = g_strdup(class_attr->value);
440 | /* reorder the attributes mnt-by and nic-hdl come first */
441 | /*g_slist_foreach(old_attr_list, each_attribute_1_pass, parse);
442 | g_slist_free(old_attr_list);
443 | old_attr_list = parse->new_attr_list; */
444 | parse->new_attr_list = NULL;
445 | /* assign auto nic-hdl and slit attributes */
446 | g_slist_foreach(old_attr_list, each_attribute_2_pass, parse);
447 | g_slist_free(old_attr_list);
448 | parse->obj->attributes = parse->new_attr_list;
449 |
450 | return(1);
451 | }
452 |
453 |
454 | /******************************************************************
455 | * report_transaction() *
456 | * *
457 | * Prints error report to the log *
458 | * *
459 | * reason - additional message that will be included *
460 | * *
461 | * *****************************************************************/
462 | static int report_transaction(Transaction_t *tr, long transaction_id, Log_t *log, ut_timer_t *psotime, char *reason)
463 | {
464 | int result=0;
465 | ut_timer_t fotime;
466 | float timediff;
467 | const char *class_name = DF_class_type2name(tr->class_type);
468 | char *primary_key = tr->K->str;
469 |
470 |
471 | /* calculate statistics */
472 | UT_timeget(&fotime);
473 | timediff = UT_timediff(psotime, &fotime);
474 |
475 | if(tr->succeeded==0) {
476 | result=tr->error;
477 | log->num_failed++;
478 | ER_inf_va(FAC_UD, ASP_UD_UPDLOG, "[%ld] %.2fs FAILED [%s:%s][%s]", transaction_id, timediff, class_name, primary_key, reason);
479 | /* ER_inf_va(FAC_UD, ASP_UD_UPDLOG, "[%ld] object: FAILED [%s][%s](%d/%d)", transaction_id, , reason, log->num_failed, (log->num_failed)+(log->num_ok)); */
480 | if(result & ERROR_U_OBJ) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: referential integrity error", transaction_id);
481 | if(result & ERROR_U_AUT) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: authentication error", transaction_id);
482 | if(result & ERROR_U_BADOP) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: unsupported operation", transaction_id);
483 | if(result & ERROR_U_COP) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: conflicting operation", transaction_id);
484 | if(result & ERROR_U_NSUP) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: this type is not supported", transaction_id);
485 | ER_dbg_va(FAC_UD, ASP_UD_OBJ, "[%ld] object: details [%s]", transaction_id, (tr->error_script)->str);
486 | result=1; /* # of failures */
487 | }
488 | else {
489 | result=0;
490 | log->num_ok++;
491 | /* ER_inf_va(FAC_UD, ASP_UD_OBJ, "[%ld] object: OK [%s](%d/%d)", transaction_id, obj_name, log->num_ok, (log->num_failed)+(log->num_ok)); */
492 | ER_inf_va(FAC_UD, ASP_UD_OBJ, "[%ld] %.2fs OK [%s:%s][%s]", transaction_id, timediff, class_name, primary_key, reason);
493 | ER_dbg_va(FAC_UD, ASP_UD_OBJ, "[%ld] object: details [%s]", transaction_id, (tr->error_script)->str);
494 | }
495 |
496 | return(result);
497 | }/* report_transaction() */
498 |
499 |
500 |
501 | /************************************************************
502 | * process_nrtm() *
503 | * *
504 | * Process object in NRTM client mode *
505 | * *
506 | * nrtm - pointer to _nrtm structure *
507 | * log - pointer to Log_t structure *
508 | * object_name - name of the object *
509 | * operation - operation code (OP_ADD/OP_DEL) *
510 | * *
511 | * Returns: *
512 | * 1 - okay *
513 | * <0 - error *
514 | * *
515 | ************************************************************/
516 |
517 | static int process_nrtm(UD_stream_t *ud_stream, Transaction_t *tr, int operation)
518 | {
519 | int result=0;
520 | int dummy=0;
521 | struct _nrtm *nrtm = ud_stream->nrtm;
522 | long serial_id;
523 | Log_t *log_ptr= &(ud_stream->log);
524 | ut_timer_t sotime;
525 | int ta_upd_nhr;
526 |
527 | /* Start timer for statistics */
528 | UT_timeget(&sotime);
529 |
530 | /* We allow NRTM updates for some inconsistent objects */
531 | /* One of the examples is reference by name which looks like nic-handle */
532 | /* For this purpose we allow dummy creation when updating an object */
533 | /* We also check for dummy allowance when deleting an object */
534 | /* this is done to allow deletion of person objects referenced by name */
535 |
536 | tr->mode|=B_DUMMY;
537 | if(IS_NO_NHR(tr->mode))ta_upd_nhr=0; else ta_upd_nhr = TA_UPD_NHR;
538 |
539 | switch (operation) {
540 |
541 | case OP_ADD:
542 | if(nrtm->tr){ /* DEL ADD => saved*/
543 | if(tr->object_id==0) {
544 | /* object does not exist in the DB */
545 | /* delete the previous(saved) object*/
546 | object_process(nrtm->tr);
547 | /* create DEL serial */
548 | UD_lock_serial(nrtm->tr);
549 | serial_id = UD_create_serial(nrtm->tr);
550 | CP_CREATE_S_PASSED(nrtm->tr->action); TR_update_status(nrtm->tr);
551 | UD_commit_serial(nrtm->tr);
552 | UD_unlock_serial(nrtm->tr);
553 | /* Mark TR as clean */
554 | TR_mark_clean(nrtm->tr);
555 | /* log the transaction */
556 | result=report_transaction(nrtm->tr, serial_id, log_ptr, &sotime, "M:DEL");
557 |
558 | object_free(nrtm->tr->object);
559 | transaction_free(nrtm->tr); nrtm->tr=NULL;
560 |
561 | /* Create an object and update NHR */
562 | tr->action=(TA_CREATE | ta_upd_nhr);
563 | /* restart the timer for statistics */
564 | UT_timeget(&sotime);
565 | object_process(tr); /* create a new one*/
566 | /* create ADD serial */
567 | UD_lock_serial(tr);
568 | serial_id = UD_create_serial(tr);
569 | CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
570 | UD_commit_serial(tr);
571 | UD_unlock_serial(tr);
572 | /* Mark TR as clean */
573 | TR_mark_clean(tr);
574 | /* log the transaction */
575 | result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD");
576 | }
577 | else {
578 | /* object already exists in the DB - update or dummy replacement*/
579 | /*compare the two, may be we may collapse operations*/
580 | if(tr->object_id==nrtm->tr->object_id) {
581 | /* DEL-ADD ->> UPDATE */
582 | object_free(nrtm->tr->object);
583 | transaction_free(nrtm->tr); nrtm->tr=NULL;
584 | tr->action=TA_UPD_CLLPS;
585 | object_process(tr);
586 | /* create DEL+ADD serial records */
587 | UD_lock_serial(tr);
588 | tr->action=TA_DELETE; serial_id = UD_create_serial(tr);
589 | result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:DEL(UPD)");
590 |
591 | /* restart the timer for statistics */
592 | UT_timeget(&sotime);
593 | tr->sequence_id++;
594 | tr->action=TA_CREATE; serial_id = UD_create_serial(tr);
595 | CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
596 | UD_commit_serial(tr);
597 | UD_unlock_serial(tr);
598 | /* Mark TR as clean */
599 | TR_mark_clean(tr);
600 | result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD(UPD)");
601 | }
602 | else { /* this should be a dummy object in the database(that we are going to replace with the real one */
603 | /* or an interleaved operation*/
604 | object_process(nrtm->tr); /* delete the previous(saved) object*/
605 | /* create a DEL serial record */
606 | UD_lock_serial(nrtm->tr);
607 | serial_id = UD_create_serial(nrtm->tr);
608 | CP_CREATE_S_PASSED(nrtm->tr->action); TR_update_status(nrtm->tr);
609 | UD_commit_serial(nrtm->tr);
610 | UD_unlock_serial(nrtm->tr);
611 | /* Mark TR as clean */
612 | TR_mark_clean(nrtm->tr);
613 | /* result=report_transaction(nrtm->tr, log_ptr, nrtm->object_name, "NRTM:DEL:While deleting previous(saved)");*/
614 | /* log the transaction */
615 | result=report_transaction(nrtm->tr, serial_id, log_ptr, &sotime, "M:DEL");
616 |
617 |
618 | object_free(nrtm->tr->object);
619 | transaction_free(nrtm->tr); nrtm->tr=NULL;
620 |
621 | /* restart the timer for statistics */
622 | UT_timeget(&sotime);
623 |
624 | tr->action=TA_UPDATE;
625 | /* check if we are replacing a dummy object */
626 | dummy=isdummy(tr);
627 | /* If we are replacing dummy with a real object update NHR */
628 | if(dummy==1) tr->action = (ta_upd_nhr | TA_UPD_DUMMY);
629 | /* fprintf(stderr,"UPDATE next(dummy)\n"); */
630 | object_process(tr); /* create a new one*/
631 | /* result=report_transaction(tr, log_ptr, object_name, "NRTM:ADD:While creating new"); */
632 | /* For serials this is CREATE operation. Increase sequence to have it correct in serals */
633 | if(dummy==1) { tr->action=TA_CREATE; tr->sequence_id++; }
634 | /* create ADD serial record */
635 | UD_lock_serial(tr);
636 | serial_id = UD_create_serial(tr);
637 | CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
638 | UD_commit_serial(tr);
639 | UD_unlock_serial(tr);
640 | /* Mark TR as clean */
641 | TR_mark_clean(tr);
642 | /* log the transaction */
643 | result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD");
644 |
645 | }
646 | }
647 | }
648 | else { /* ADD ADD =>brand new object*/
649 | if(tr->object_id==0) {
650 | /* fprintf(stderr,"CREATE new\n");*/
651 | /* Create an object and update NHR */
652 | tr->action=(TA_CREATE | ta_upd_nhr);
653 | object_process(tr);
654 | /* create ADD serial */
655 | UD_lock_serial(tr);
656 | serial_id = UD_create_serial(tr);
657 | CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
658 | UD_commit_serial(tr);
659 | UD_unlock_serial(tr);
660 |
661 | /* Mark TR as clean */
662 | TR_mark_clean(tr);
663 | /* log the transaction */
664 | result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD");
665 | }
666 | else { /* object already exists in the database */
667 | /* this may happen because of dummies*/
668 | /* or with some implementations of mirroring protocol that have atomic update */
669 | /* instead of add + del */
670 | tr->action=TA_UPDATE;
671 | dummy=isdummy(tr);
672 | /* If we are replacing dummy with a real object update NHR */
673 | if(dummy==1) tr->action = ( ta_upd_nhr| TA_UPD_DUMMY);
674 | object_process(tr);
675 | /* For serials this is CREATE operation. Increase sequence to have it correct in serals */
676 | if(dummy==1) { tr->action=TA_CREATE; tr->sequence_id++; }
677 | /* create ADD serial record */
678 | UD_lock_serial(tr);
679 | serial_id = UD_create_serial(tr);
680 | CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
681 | UD_commit_serial(tr);
682 | UD_unlock_serial(tr);
683 | /* Mark TR as clean */
684 | TR_mark_clean(tr);
685 | /* log the transaction */
686 | result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD");
687 |
688 | }
689 | }
690 | break;
691 |
692 | case OP_DEL:
693 | if(nrtm->tr){ /*DEL DEL =>saved */
694 | object_process(nrtm->tr); /* delete the previous(saved) object*/
695 | /* create DEL serial record */
696 | UD_lock_serial(nrtm->tr);
697 | serial_id = UD_create_serial(nrtm->tr);
698 | CP_CREATE_S_PASSED(nrtm->tr->action); TR_update_status(nrtm->tr);
699 | UD_commit_serial(nrtm->tr);
700 | UD_unlock_serial(nrtm->tr);
701 | /* Mark TR as clean */
702 | TR_mark_clean(nrtm->tr);
703 | /* log the transaction */
704 | result=report_transaction(nrtm->tr, serial_id, log_ptr, &sotime, "M:DEL");
705 |
706 | object_free(nrtm->tr->object);
707 | transaction_free(nrtm->tr); nrtm->tr=NULL;
708 | }
709 | /* save the real object (not a dummy one ) */
710 | if(tr->object_id>0 && !isdummy(tr)){
711 | /* save the object*/
712 | tr->action=(TA_DELETE | ta_upd_nhr);
713 | nrtm->tr=tr;
714 | return(0);
715 | }
716 | else { /* this is an error - Trying to DEL non-existing object*/
717 | tr->succeeded=0; tr->error|=ERROR_U_COP;
718 | tr->action=(TA_DELETE | ta_upd_nhr);
719 | /* create and initialize TR record for crash recovery */
720 | TR_create_record(tr);
721 | /* create DEL serial record anyway */
722 | UD_lock_serial(tr);
723 | serial_id = UD_create_serial(tr);
724 | CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
725 | UD_commit_serial(tr);
726 | UD_unlock_serial(tr);
727 | /* Mark TR as clean */
728 | TR_mark_clean(tr);
729 | /* log the transaction */
730 | result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:DEL: non-existing object");
731 |
732 | }
733 | break;
734 |
735 | default:
736 | tr->succeeded=0; tr->error |=ERROR_U_BADOP;
737 | break;
738 | }
739 |
740 | /* Free resources */
741 | object_free(tr->object);
742 | transaction_free(tr);
743 |
744 | return(result);
745 | } /* process_nrtm() */
746 |
747 |
748 |
749 | /************************************************************
750 | * process_updates() *
751 | * *
752 | * Process object in update mode *
753 | * *
754 | * ud_stream - pointer to UD_stream structure *
755 | * object_name - name of the object *
756 | * operation - operation code (OP_ADD/OP_DEL) *
757 | * *
758 | * Note: *
759 | * Frees tr and tr->obj on exit *
760 | * *
761 | * Returns: *
762 | * 0 - okay *
763 | * <0- number of failed objects *
764 | * *
765 | ************************************************************/
766 |
767 | static int process_updates(UD_stream_t *ud_stream, Transaction_t *tr, int operation)
768 | {
769 | int result=0;
770 | Log_t *log_ptr= &(ud_stream->log);
771 | int dummy=0;
772 | ut_timer_t sotime;
773 | int ta_upd_nhr;
774 | char *reason;
775 |
776 | /* Start timer for statistics */
777 | UT_timeget(&sotime);
778 | if(IS_NO_NHR(tr->mode))ta_upd_nhr=0; else ta_upd_nhr = TA_UPD_NHR;
779 |
780 | switch(operation) {
781 | /* Compare operations and report an error if they do not match */
782 | case OP_ADD:
783 | if(tr->object_id!=0) { /* trying to create, but object exists */
784 | tr->succeeded=0; tr->error|=ERROR_U_COP;
785 | reason="U:ADD:object already exists";
786 | UD_ack(tr); /* Send a NACK */
787 | } else {
788 | /* Action: create the object and update NHR */
789 | tr->action=(TA_CREATE | ta_upd_nhr);
790 | reason="U:ADD";
791 | object_process(tr);
792 | }
793 | break;
794 | case OP_UPD:
795 | if(tr->object_id==0) { /* trying to update non-existing object*/
796 | tr->succeeded=0; tr->error|=ERROR_U_COP;
797 | reason="U:UPD:non-existing object";
798 | UD_ack(tr); /* Send a NACK */
799 | } else {
800 | tr->action=TA_UPDATE;
801 | reason="U:UPD";
802 | dummy=isdummy(tr);
803 | /* If we are replacing dummy with a real object update NHR */
804 | if(dummy==1) tr->action = (ta_upd_nhr | TA_UPD_DUMMY);
805 | object_process(tr);
806 | }
807 | break;
808 |
809 | case OP_DEL:
810 | if(tr->object_id==0) { /* trying t delete non-existing object*/
811 | tr->succeeded=0; tr->error|=ERROR_U_COP;
812 | reason="U:DEL:non-existing object";
813 | UD_ack(tr);
814 | } else {
815 | tr->action=(TA_DELETE | ta_upd_nhr);
816 | reason="U:DEL";
817 | object_process(tr);
818 | }
819 | break;
820 |
821 | default:
822 | /* bad operation for this mode if not standalone */
823 | if(IS_STANDALONE(tr->mode)) {
824 | if(tr->object_id==0){
825 | tr->action=(TA_CREATE | ta_upd_nhr);
826 | reason="U:ADD";
827 | }
828 | else {
829 | tr->action=TA_UPDATE;
830 | reason="U:UPD";
831 | }
832 | object_process(tr);
833 | }
834 | else {
835 | tr->succeeded=0;
836 | tr->error|=ERROR_U_BADOP;
837 | reason="U:bad operation";
838 | UD_ack(tr); /* Send a NACK */
839 | }
840 | break;
841 | }
842 | /* If not in standalone mode create serial and copy error transcript */
843 | if(!IS_STANDALONE(tr->mode)) {
844 | if(tr->succeeded){
845 | /* we don't want to generate DEL serial for dummy replacement*/
846 | if(dummy==1) { tr->action=TA_CREATE; tr->sequence_id++; }
847 | UD_lock_serial(tr);
848 | UD_create_serial(tr);
849 | CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
850 | UD_commit_serial(tr);
851 | UD_unlock_serial(tr);
852 | /* Mark the TR as clean */
853 | TR_mark_clean(tr);
854 | }
855 | }
856 |
857 | /* Make a report. U stands for update stream. No reason */
858 | result=report_transaction(tr, tr->transaction_id, log_ptr, &sotime, reason);
859 |
860 | /* Free resources */
861 | object_free(tr->object);
862 | transaction_free(tr);
863 |
864 | return(result);
865 |
866 | } /* process_updates() */
867 |
868 |
869 | /************************************************************
870 | * *
871 | * int process_transaction() *
872 | * *
873 | * Processes the transaction *
874 | * *
875 | * ud_stream - pointer to UD_stream_t structure *
876 | * *
877 | * Returns: *
878 | * 0 - no error *
879 | * <0- number of failed objects *
880 | * *
881 | ************************************************************/
882 |
883 | /* It frees the obj */
884 |
885 | static int process_transaction(UD_stream_t *ud_stream,
886 | Obj_parse_t *parse,
887 | int operation,
888 | long transaction_id)
889 | {
890 | Transaction_t *tr = NULL;
891 | Object_t *obj = parse->obj;
892 | int result;
893 |
894 | /* check if the requested transaction has already been processed */
895 | /* this may happen in case of crash. If so, just send an ack and return */
896 | if(TR_check(ud_stream->db_connection, transaction_id, (ud_stream->condat).sock))return(1);
897 |
898 | /* start new transaction now */
899 | tr = transaction_new(ud_stream->db_connection, obj->type);
900 |
901 | /* Return with error if transaction cannot be created */
902 | if (tr == NULL) die;
903 |
904 | tr->mode=ud_stream->ud_mode;
905 | tr->load_pass=ud_stream->load_pass;
906 | tr->object=obj;
907 | tr->nh=parse->nh_ptr;
908 | tr->source_hdl=ud_stream->source_hdl;
909 | tr->socket=(ud_stream->condat).sock;
910 | tr->transaction_id=transaction_id;
911 |
912 | /* We perform no commit/rollback in the loader mode, so thread_id should be set to 0 */
913 | if(ud_stream->load_pass!=0) { tr->thread_ins=0; tr->thread_upd=0; }
914 |
915 | /* For the first load pass we only create objects */
916 | if(ud_stream->load_pass==1) {
917 | Object_t *obj=tr->object;
918 | /* still we need to fill tr->K (last.pkey) field */
919 | g_slist_foreach(obj->attributes, ud_each_primary_key_select, tr);
920 | tr->object_id=0;
921 | }
922 | else tr->object_id=get_object_id(tr);
923 |
924 | /* Object cannot be retrieved */
925 | if(tr->object_id==-1) { /* DB error*/
926 | tr->succeeded=0;
927 | tr->error |= ERROR_U_DBS;
928 | ER_perror(FAC_UD, UD_SQL, "%s: Object cannot be retrieved", parse->object_name);
929 | die;
930 | transaction_free(tr);
931 | object_free(obj);
932 | }
933 | /* save the name of person/role as we need it for referential */
934 | /* integrity check when deleting the object against names. */
935 | /* This is needed to support legacy references by name rather */
936 | /* then by nic_hdl */
937 | if((tr->class_type==C_PN) || (tr->class_type==C_RO)){
938 | /* Save the value */
939 | tr->save=g_strdup(parse->object_name);
940 | /* attribute_free(attr, NULL); */
941 | }
942 |
943 | /* Process transaction. tr and obj are freed inside the process_* functions */
944 |
945 | if(IS_UPDATE(ud_stream->ud_mode))
946 | /* We are in update mode */
947 | result=process_updates(ud_stream, tr, operation);
948 | else
949 | /* We are in NRTM mode */
950 | result=process_nrtm(ud_stream, tr, operation);
951 |
952 | return(result);
953 |
954 | }
955 |
956 |
957 | /************************************************************
958 | * *
959 | * int UD_process_stream(UD_stream_t *ud_stream) *
960 | * *
961 | * Processes the stream *
962 | * *
963 | * ud_stream - pointer to UD_stream_t structure *
964 | * *
965 | * Returns: *
966 | * in update mode (!standalone)(1 object processed): *
967 | * 1 - no error *
968 | * <0- errors *
969 | * *
970 | * in NRTM & standalone modes *
971 | * total number of object processed *
972 | * *
973 | ************************************************************/
974 |
975 | int UD_process_stream(UD_stream_t *ud_stream)
976 | {
977 | char line_buff[STR_XXL];
978 | Object_t *obj = NULL;
979 | SQ_connection_t *sql_connection;
980 | int start_object;
981 | int a_type;
982 | struct _nrtm *nrtm;
983 | Log_t *log_ptr= &(ud_stream->log);
984 | ut_timer_t stime, ftime, sotime;
985 | float obj_second1, obj_second10, timediff;
986 | int result;
987 | int operation=0;
988 | int interrupt=0;
989 | int do_update;
990 | int default_ud_mode = ud_stream->ud_mode;
991 | Line_Type_t linetype;
992 | Transaction_t *tr;
993 | long transaction_id=0; /* transaction_id (to be supplied by DBupdate and stored in Database) */
994 | long serial_id;
995 | Obj_parse_t obj_parse; /* the structure used to parse a text object */
996 |
997 |
998 |
999 |
1000 | nrtm=ud_stream->nrtm;
1001 | start_object = 1;
1002 | a_type=-1;
1003 |
1004 |
1005 | /* Check connection to the database */
1006 | if(SQ_ping(ud_stream->db_connection)) {
1007 | ER_perror(FAC_UD, UD_SQL, "%s", SQ_error(ud_stream->db_connection));
1008 | die;
1009 | }
1010 |
1011 | sql_connection=ud_stream->db_connection;
1012 |
1013 | ud_parse_init(sql_connection, &obj_parse);
1014 | /* Start timer for statistics */
1015 | UT_timeget(&stime);
1016 |
1017 | /* Main loop. Reading input stream line by line */
1018 | /* Empty line signals to start processing an object, if we have it */
1019 | while (SK_cd_gets(&ud_stream->condat, line_buff, sizeof(line_buff))>0) {
1020 |
1021 |
1022 | switch (linetype=line_type(line_buff, &transaction_id)) {
1023 | case LINE_ATTRIBUTE:
1024 | /* parse the object line by line */
1025 | obj = UD_parse_object(&obj_parse, line_buff);
1026 |
1027 | break;
1028 |
1029 | case LINE_COMMENT:
1030 | break;
1031 |
1032 | case LINE_EOF:
1033 | break;
1034 |
1035 | case LINE_ACK:
1036 | tr = transaction_new(ud_stream->db_connection, 0);
1037 | tr->transaction_id=transaction_id;
1038 | TR_delete_record(tr);
1039 | transaction_free(tr);
1040 | break;
1041 |
1042 |
1043 | case LINE_ADD:
1044 | /* restore the default operation mode */
1045 | operation=OP_ADD;
1046 | ud_stream->ud_mode=default_ud_mode;
1047 | break;
1048 |
1049 | case LINE_OVERRIDE_ADD:
1050 | /* for override - switch the dummy bit on */
1051 | operation=OP_ADD;
1052 | ud_stream->ud_mode=default_ud_mode|B_DUMMY;
1053 | break;
1054 |
1055 | case LINE_UPD:
1056 | /* restore the default operation mode */
1057 | operation=OP_UPD;
1058 | ud_stream->ud_mode=default_ud_mode;
1059 | break;
1060 |
1061 | case LINE_OVERRIDE_UPD:
1062 | /* for override - switch the dummy bit on */
1063 | operation=OP_UPD;
1064 | ud_stream->ud_mode=default_ud_mode|B_DUMMY;
1065 | break;
1066 |
1067 | case LINE_DEL:
1068 | /* restore the default operation mode */
1069 | operation=OP_DEL;
1070 | ud_stream->ud_mode=default_ud_mode;
1071 | break;
1072 |
1073 | case LINE_OVERRIDE_DEL:
1074 | /* for override - switch the dummy bit on */
1075 | operation=OP_DEL;
1076 | ud_stream->ud_mode=default_ud_mode|B_DUMMY;
1077 | break;
1078 |
1079 | case LINE_EMPTY:
1080 | /* start processing the object */
1081 | if ((obj=obj_parse.obj)) { /* if not just garbage*/
1082 | /* normalize the object (reorder and split attributes */
1083 | ud_normalize(&obj_parse);
1084 | /* XXX */
1085 | /* print_object(obj); */
1086 |
1087 | /* start new transaction now */
1088 | ER_dbg_va(FAC_UD, ASP_UD_OBJ, "[%ld] object: [%s] ", transaction_id, obj_parse.object_name);
1089 |
1090 | /* fprintf(stderr, "transction # %ld\n", transaction_id); */
1091 | result=process_transaction(ud_stream, &obj_parse, operation, transaction_id);
1092 | /* process_transaction() frees tr and obj structures, */
1093 | /* so make sure we'll not reference these objects in the future */
1094 | operation=OP_NOOP;
1095 | transaction_id=0;
1096 | ud_stream->ud_mode=default_ud_mode;
1097 | ud_parse_free(&obj_parse);
1098 |
1099 | /* this is a good place for quick interrupt */
1100 | do_update=CO_get_do_update();
1101 | if (do_update) interrupt=0; else interrupt=1;
1102 | } /* if this is a real object */
1103 | /* initialize the parsing structure */
1104 | ud_parse_init(sql_connection, &obj_parse);
1105 |
1106 | break;
1107 |
1108 | default:
1109 | die;
1110 | } /* switch */
1111 |
1112 | /* Finish processing if interrupt has been set */
1113 | if (interrupt) break;
1114 | } /* Main loop of data stream processing : while */
1115 |
1116 | /* Some postprocessing */
1117 | if(IS_NRTM_CLNT(ud_stream->ud_mode)){
1118 | /* We are in NRTM mode */
1119 | /* Clean up */
1120 | /* fclose(ud_stream->stream); */
1121 | /* In NRTM mode there may be a saved object that is unprocessed */
1122 | if(nrtm->tr){ /*saved backlog?*/
1123 | /* restart the timer for statistics */
1124 | UT_timeget(&sotime);
1125 | object_process(nrtm->tr); /* delete the previous(saved) object*/
1126 | /* result=report_transaction(nrtm->tr, &(ud_stream->log), nrtm->object_name,
1127 | "NRTM:DEL:While deleting previous(saved) object"); */
1128 | /* create DEL serial record no matter what the result is */
1129 | UD_lock_serial(nrtm->tr);
1130 | serial_id = UD_create_serial(nrtm->tr);
1131 | CP_CREATE_S_PASSED(nrtm->tr->action); TR_update_status(nrtm->tr);
1132 | UD_commit_serial(nrtm->tr);
1133 | UD_unlock_serial(nrtm->tr);
1134 | /* Mark TR as clean */
1135 | TR_mark_clean(nrtm->tr);
1136 | /* log the transaction */
1137 | result=report_transaction(nrtm->tr, serial_id, log_ptr, &sotime, "M:DEL");
1138 |
1139 | object_free(nrtm->tr->object);
1140 | transaction_free(nrtm->tr); nrtm->tr=NULL;
1141 | }
1142 | }
1143 |
1144 | /* That's all. Free GString */
1145 | /* g_string_free(g_line_buff, TRUE);*/
1146 |
1147 |
1148 | /* Calculate some statistics */
1149 | /* ftime=time(NULL); */
1150 | UT_timeget(&ftime);
1151 | /* obj_second1 = (float)(log_ptr->num_ok)/(ftime-stime);
1152 | obj_second10 = (float)(log_ptr->num_ok+log_ptr->num_failed)/(ftime-stime); */
1153 | timediff = UT_timediff(&stime, &ftime);
1154 | obj_second1 = (float)(log_ptr->num_ok)/timediff;
1155 | obj_second10 = (float)(log_ptr->num_ok+log_ptr->num_failed)/timediff;
1156 |
1157 | /* Print the report */
1158 | if(IS_STANDALONE(ud_stream->ud_mode) || (!IS_UPDATE(ud_stream->ud_mode))) {
1159 |
1160 | ER_inf_va(FAC_UD, ASP_UD_UPDLOG,"%s ******** report **********", UD_TAG);
1161 | ER_inf_va(FAC_UD, ASP_UD_UPDLOG,"%s %d objects OK (%7.4f obj/s)", UD_TAG, log_ptr->num_ok, obj_second1);
1162 | ER_inf_va(FAC_UD, ASP_UD_UPDLOG,"%s %d objects failed", UD_TAG, log_ptr->num_failed);
1163 | ER_inf_va(FAC_UD, ASP_UD_UPDLOG,"%s average processing time %7.4f obj/s (%6.2f obj/min)", UD_TAG,
1164 | obj_second10, obj_second10*60);
1165 | result=log_ptr->num_ok+log_ptr->num_failed;
1166 | }
1167 | return(result);
1168 |
1169 | } /* UD_process_stream */
1170 |