utils/hs_cleanup/hs_cleanup.c

/* [<][>]
[^][v][top][bottom][index][help] */

FUNCTIONS

This source file includes following functions.
  1. archive_history
  2. archive_last
  3. delete_history_entry
  4. delete_last_entry
  5. update_prev_serial_of_next_object
  6. update_serial_in_last
  7. update_serial_in_history
  8. main
  9. create_archive_tables
  10. create_auxiliary_table
  11. create_table
  12. find_unreferenced_history_entries
  13. delete_unreferenced_history_entries
  14. PushTblObjList
  15. get_highest_serial_before_date
  16. archive_serials_and_history
  17. lock_last_history_serial_tables
  18. unlock_all_tables
  19. update_hs_auxiliary_checkpoint
  20. archive_serial
  21. archive_failed_transaction
  22. copy_into_history_archive
  23. copy_deleted_object_into_history_archive
  24. update_history_sequence_id_and_timestamp
  25. check_if_next_is_deletion
  26. update_prev_serial_of_object
  27. update_serial_of_object
  28. delete_serial_entry
  29. delete_failed_transaction_entry
  30. delete_entry_from_object_table
  31. fetch_timestamp_from_last
  32. optimize_sql_table
  33. execute_sql_query
  34. execute_sql_command
  35. usage

   1 /***************************************
   2   $Revision: 1.22 $
   3 
   4   History/Serial Cleanup (hs_cleanup). This utility archives serials
   5   and history entries, and deletes them from the live database.
   6 
   7   Status: NOT COMPLETE, NOT REVUED, NOT FULLY TESTED
   8 
   9   ******************/ /******************
  10   Filename            : hs_cleanup.c
  11   Authors             : daniele@ripe.net
  12   OSs Tested          : Solaris 7
  13   ******************/ /******************
  14   Copyright (c) 2000                              RIPE NCC
  15  
  16   All Rights Reserved
  17   
  18   Permission to use, copy, modify, and distribute this software and its
  19   documentation for any purpose and without fee is hereby granted,
  20   provided that the above copyright notice appear in all copies and that
  21   both that copyright notice and this permission notice appear in
  22   supporting documentation, and that the name of the author not be
  23   used in advertising or publicity pertaining to distribution of the
  24   software without specific, written prior permission.
  25   
  26   THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING
  27   ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS; IN NO EVENT SHALL
  28   AUTHOR BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY
  29   DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
  30   AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  31   OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  32   ***************************************/
  33 
  34 #include <stdio.h>
  35 #include <stdlib.h>
  36 #include <string.h>
  37 #include <sys/time.h>
  38 
  39 #include "mysql_driver.h"
  40 #include "stubs.h"
  41 
  42 
  43 
  44 #define MYSQL_HOST "myhost.mydb.net"
  45 #define MYSQL_PORT 3306
  46 #define MYSQL_USER "sqluser"
  47 #define MYSQL_PSWD "sqlpswd"
  48 #define MYSQL_DB "sqldb"
  49 
  50 /* String sizes */
  51 #define STR_S   63
  52 #define STR_M   255
  53 #define STR_L   1023
  54 #define STR_XL  4095
  55 #define STR_XXL 16383
  56 
  57 #define MAXCMDLEN 8192
  58 #define MAXNUMCMDS 65536
  59 
  60 #define DATE 966050936
  61 
  62 /* Set the default lapse before which we want to archive:
  63  * it must be in seconds.
  64  * 1min = 60s
  65  * 1hr = 3600s
  66  * 1day = 86400s
  67  * 1wk = 604800s
  68  */
  69 
  70 #define MINUTE 60
  71 #define HOUR 3600
  72 #define DAY 86400
  73 #define WEEK 604800
  74 
  75 #define DEBUG 1
  76 #define ARCHIVE_ONLY 1
  77 
  78 
  79 enum {
  80   IN_HISTORY_TABLE = 0,
  81   IN_LAST_TABLE,
  82   IN_FAILED_TRANSACTION_TABLE,
  83   IN_UNDEF_TABLE
  84 };
  85 
  86 enum {
  87   OP_NULL = 0,
  88   OP_ADD,
  89   OP_DELETE,
  90   OP_UPDATE
  91 };
  92 
  93 
  94 enum {
  95   CHKP_NOOP = 1,
  96   CHKP_ARCHIVED_SERIAL,
  97   CHKP_ARCHIVED_HISTORY,
  98   CHKP_DELETED_HISTORY,
  99   CHKP_DELETED_SERIAL,
 100   CHKP_DONE
 101 };
 102 
 103 typedef struct table_object *tblobjPtr;
 104 
 105 typedef struct table_object {
 106   int objid;
 107   int seqid;
 108   tblobjPtr next;
 109 } tblObjList;
 110 
 111 
 112 /* Global variables */
 113 
 114 SQ_connection_t *connection;
 115 int debug = DEBUG;
 116 int archive_only = ARCHIVE_ONLY;
 117 
 118 /* Function definitions */
 119 
 120 int main (int argc, char *argv[]);
 121 void usage(char *argv[]);
 122 
 123 static tblObjList *find_unreferenced_history_entries(int date);
 124 static int delete_unreferenced_history_entries(tblObjList *objectsToDelete);
 125 static int create_auxiliary_table();
 126 static int create_archive_tables();
 127 static int create_table(const char *cmd);
 128 static int get_highest_serial_before_date(int *highest_serial_ptr, int date);
 129 static int archive_serials_and_history(int highest_serial);
 130 static int archive_serial(int ser_id, int obj_id, int seq_id, int op);
 131 static int archive_failed_transaction(int ser_id);
 132 static int copy_into_history_archive(int ser_id, int obj_id, int seq_id, const char *tablename);
 133 static int copy_deleted_object_into_history_archive (int ser_id, int obj_id, int seq_id, const char *tablename);
 134 static int update_history_sequence_id_and_timestamp(int ser_id, int new_seq_id, int new_timestamp);
 135 static int delete_serial_entry(int ser_id);
 136 static int delete_failed_transaction_entry (int ser_id);
 137 static int delete_entry_from_object_table(int obj_id, int seq_id, const char* tablename);
 138 static int fetch_timestamp_from_last(int obj_id, int ser_id);
 139 static int execute_sql_command(const char *cmd);
 140 static int execute_sql_query(const char *query, SQ_result_set_t **result_ptr);
 141 static int check_if_next_is_deletion (int obj_id, int seq_id);
 142 static int update_prev_serial_of_object (int obj_id, int seq_id, int prev_ser, const char *tablename);
 143 static int update_serial_of_object (int obj_id, int seq_id, int ser_id, const char *tablename);
 144 static int lock_last_history_serial_tables();
 145 static int unlock_all_tables();
 146 static int optimize_sql_table(const char* tablename);
 147 static int update_hs_auxiliary_checkpoint(int ser_id, int obj_id, int seq_id, int timestamp, int checkpoint);
 148 
 149 
 150 
 151 #define archive_history(ser_id, obj_id, seq_id) copy_into_history_archive(ser_id, obj_id, seq_id, "history")
     /* [<][>][^][v][top][bottom][index][help] */
 152 #define archive_last(ser_id, obj_id, seq_id) copy_into_history_archive(ser_id, obj_id, seq_id, "last")
     /* [<][>][^][v][top][bottom][index][help] */
 153 #define delete_history_entry(obj_id, seq_id) delete_entry_from_object_table(obj_id, seq_id, "history")
     /* [<][>][^][v][top][bottom][index][help] */
 154 #define delete_last_entry(obj_id, seq_id) delete_entry_from_object_table(obj_id, seq_id, "last")
     /* [<][>][^][v][top][bottom][index][help] */
 155 #define update_prev_serial_of_next_object(obj_id, seq_id, prev_ser, tablename) update_prev_serial_of_object(obj_id, seq_id+1, prev_ser, tablename)
     /* [<][>][^][v][top][bottom][index][help] */
 156 #define update_serial_in_last(obj_id, seq_id, ser_id) update_serial_of_object(obj_id, seq_id, ser_id, "last")
     /* [<][>][^][v][top][bottom][index][help] */
 157 #define update_serial_in_history(obj_id, seq_id, ser_id) update_serial_of_object(obj_id, seq_id, ser_id, "history")
     /* [<][>][^][v][top][bottom][index][help] */
 158 
 159 
 160 
 161 /* XXX Fixme:
 162    - No more hardcoded variables
 163    - Subroutine (+option) to warn that the size is bigger than a watermark
 164    - Checkpointing for crash recovery
 165    */
 166 
 167 
 168 
 169 int main (int argc, char *argv[])
     /* [<][>][^][v][top][bottom][index][help] */
 170 {
 171 
 172   int ch, rc;
 173   int highest_serial;
 174   short errflg = 1;
 175   short tflg = 0;
 176   extern char *optarg;
 177   extern int optind;
 178   time_t now = time(0);
 179   time_t date;
 180   time_t lapse;
 181 
 182   char sqlhost[STR_M] = MYSQL_HOST;
 183   int sqlport = MYSQL_PORT;
 184   char sqluser[STR_M] = MYSQL_USER;
 185   char sqlpswd[STR_M] = MYSQL_PSWD;
 186   char sqldb[STR_M] = MYSQL_DB;
 187 
 188   tblObjList *objectsToDelete;
 189   tblObjList *tmpObj;
 190 
 191   /* Get options */
 192 
 193   while ((ch = getopt(argc, argv, "?T:S:M:H:D:W:h:P:u:p:d:")) != EOF )
 194     switch((char)ch)
 195       {
 196       case 'T':
 197         if (tflg)
 198           errflg++;
 199         else
 200           {
 201             tflg++;
 202             errflg = 0;
 203             date = atol (optarg);
 204           }
 205         break;
 206       case 'S':
 207         if (tflg)
 208           errflg++;
 209         else
 210           {
 211             tflg++;
 212             errflg = 0;
 213             lapse = atol (optarg);
 214             date = now - lapse;
 215           }
 216         break;
 217       case 'M':
 218         if (tflg)
 219           errflg++;
 220         else
 221           {
 222             tflg++;
 223             errflg = 0;
 224             lapse = atol (optarg);
 225             date = now - lapse * MINUTE;
 226           }
 227         break;
 228       case 'H':
 229         if (tflg)
 230           errflg++;
 231         else
 232           {
 233             tflg++;
 234             errflg = 0;
 235             lapse = atol (optarg);
 236             date = now - lapse * HOUR;
 237           }
 238         break;
 239       case 'D':
 240         if (tflg)
 241           errflg++;
 242         else
 243           {
 244             tflg++;
 245             errflg = 0;
 246             lapse = atol (optarg);
 247             date = now - lapse * DAY;
 248           }
 249         break;
 250       case 'W':
 251         if (tflg)
 252           errflg++;
 253         else
 254           {
 255             tflg++;
 256             errflg = 0;
 257             lapse = atol (optarg);
 258             date = now - lapse * WEEK;
 259           }
 260         break;
 261       case 'h':
 262         sprintf (sqlhost,"%s",optarg);
 263         break;
 264       case 'P':
 265         sqlport = atoi(optarg);
 266         break;
 267       case 'u':
 268         sprintf (sqluser,"%s",optarg);
 269         break;
 270       case 'p':
 271         sprintf (sqlpswd,"%s",optarg);
 272         break;
 273       case 'd':
 274         sprintf (sqldb,"%s",optarg);
 275         break;
 276       case '?':
 277       default:
 278         errflg++;
 279       }
 280 
 281 if (errflg)
 282   usage(argv);
 283 
 284 
 285   /* Initialize connection */
 286   connection = SQ_get_connection(sqlhost, sqlport,sqldb,sqluser,sqlpswd);
 287 
 288   /* Create tables for history and serials archives
 289    * if they do not exist */
 290   if ((rc = create_archive_tables()) != 0)
 291     { return(rc); }
 292 
 293   /* Create auxiliary table if it does not exist */
 294   if ((rc = create_auxiliary_table()) != 0)
 295     { return(rc); }
 296 
 297   /* XXX Call remadmin interface and stop updates */
 298 
 299 
 300   /* XXX If retcode is successful, go on */
 301 
 302   /* Deal with very old history entries, those that do not even have a corresponding
 303    * serial. These are entries which had been archived when they were in the "last" table,
 304    * and have in the meanwhile been updated. We have to:
 305    *    - Update prev_serial of their next object
 306    *    - Delete them!
 307    */
 308 
 309   /* XXX Not fully tested: some code must be updated in UD to support the extra columns
 310      serial and prev_serial */
 311   objectsToDelete = find_unreferenced_history_entries((int) date);
 312 
 313   /* printf ("Elements to be deleted:\n");
 314      for (tmpObj = objectsToDelete; tmpObj != NULL; tmpObj = tmpObj->next)
 315      {
 316      printf ("objid: %d, seqid: %d\n", tmpObj->objid, tmpObj->seqid);
 317      } */ 
 318 
 319 
 320   /* Get the biggest serial for which the history or last timestamp is lower than
 321    * the defined timestamp
 322    */
 323 
 324   if ((rc = get_highest_serial_before_date(&highest_serial, (int) date)) != 0)
 325     { return(rc); }
 326   printf ("Highest serial ID: %d\n",highest_serial);
 327   
 328 
 329   /* Execute the archiving commands */
 330 
 331   archive_serials_and_history(highest_serial);
 332 
 333   /* Optimize history serial and last tables: there might have been many deletions */
 334   optimize_sql_table("serials");
 335   optimize_sql_table("history");
 336   optimize_sql_table("last");
 337 
 338   /* XXX Call remadmin interface and restart updates */
 339 
 340 
 341   /* XXX If retcode is not successful, go on, issue a warning */
 342 
 343   /* Delete the unreferenced history entries. Must be done at the end. */
 344   /* XXX Bug here. The older entries cannot be deleted or they wreak havoc. */
 345   if (! archive_only)
 346     delete_unreferenced_history_entries(objectsToDelete);
 347 
 348   /* OK, it's over. */
 349 
 350   SQ_close_connection(connection);
 351   printf ("\nProgram done.\n");
 352   return(0);
 353 
 354 } /* main() */
 355 
 356 
 357 
 358 
 359 
 360 /* Functions */
 361 
 362 
 363 /* create_archive_tables():
 364  * Create tables for history and serials archives
 365  * if they do not exist */
 366 
 367 int create_archive_tables()
     /* [<][>][^][v][top][bottom][index][help] */
 368 {
 369 
 370   char cmd[MAXCMDLEN];
 371 
 372   sprintf (cmd, 
 373            "CREATE TABLE history_archive (
 374                 object_id int(10) unsigned DEFAULT '0' NOT NULL,
 375                 sequence_id int(10) unsigned DEFAULT '0' NOT NULL,
 376                 serial int(11) DEFAULT '0' NOT NULL,
 377                 prev_serial int(11) DEFAULT '0' NOT NULL,
 378                 timestamp int(10) unsigned DEFAULT '0' NOT NULL,
 379                 object_type tinyint(3) unsigned DEFAULT '0' NOT NULL,
 380                 object longblob NOT NULL,
 381                 PRIMARY KEY (object_id,sequence_id,serial)
 382                 );");
 383   create_table(cmd);
 384 
 385 
 386   sprintf (cmd, 
 387            "CREATE TABLE serials_archive (
 388                 serial_id int(11) DEFAULT '0' NOT NULL,
 389                 object_id int(10) unsigned DEFAULT '0' NOT NULL,
 390                 sequence_id int(10) unsigned DEFAULT '0' NOT NULL,
 391                 operation tinyint(4) unsigned DEFAULT '0' NOT NULL,
 392                 PRIMARY KEY (serial_id)
 393                 );");
 394   create_table(cmd);
 395 
 396   return(0);
 397 
 398 } /* create_archive_tables() */
 399 
 400 
 401 
 402 /* create_auxiliary_table()
 403  * This auxiliary table will record some checkpointing
 404  * data, in order to recover from crashes
 405  * and to help with the clenup of the older history tables
 406  */
 407 int create_auxiliary_table()
     /* [<][>][^][v][top][bottom][index][help] */
 408 {
 409 
 410   int state;
 411   char cmd[MAXCMDLEN];
 412 
 413   sprintf (cmd,"CREATE TABLE hs_auxiliary (
 414                 serial int(11) DEFAULT '0' NOT NULL,
 415                 object_id int(10) unsigned DEFAULT '0' NOT NULL,
 416                 sequence_id int(10) unsigned DEFAULT '0' NOT NULL,
 417                 timestamp int(10) unsigned DEFAULT '0' NOT NULL,
 418                 checkpoint tinyint(4) unsigned DEFAULT '0' NOT NULL,
 419                 PRIMARY KEY (serial)
 420                 );");
 421   state = create_table(cmd);
 422   if (state == 0) /* state != 0 only if the table already exists - other errors make the program die */
 423     {
 424       /* We also need to create a dummy row if the table had not been created */
 425       sprintf (cmd,"INSERT INTO hs_auxiliary VALUES (0, 0, 0, 0, 0)");
 426       execute_sql_command(cmd);
 427     }
 428 
 429   return(0);
 430 
 431 } /* create_auxiliary_table() */
 432 
 433 
 434 
 435 
 436 /* create_table()
 437  * This function wraps a table creation (which must be already
 438  * specified in cmd), and only issues a warning if the table
 439  * already exists.
 440  */
 441 
 442 int create_table (const char *cmd)
     /* [<][>][^][v][top][bottom][index][help] */
 443 {
 444 
 445   int state;
 446 
 447   state = SQ_execute_query(connection, cmd, NULL);
 448   if (state != 0)
 449     {
 450       /* XXX is ER_TABLE_EXISTS_ERROR mysql-bounded? */
 451       if (SQ_errno(connection) == ER_TABLE_EXISTS_ERROR)
 452         { 
 453           /* Don't die if a table already exists */
 454           fprintf (stderr,"Warning: %s\n",SQ_error(connection));
 455           return (state);
 456         }
 457       else
 458         {
 459           fprintf (stderr,"Fatal: %s\n",SQ_error(connection));
 460           dieif (state != 0);
 461         }
 462     }
 463 
 464   return(0);
 465 
 466 } /* create_table() */
 467 
 468 
 469 
 470 
 471 /* find_unreferenced_history_entries()
 472  * Deal with very old history entries, those that are not referenced by any serial,
 473  * due to a previous history/serial cleanup.
 474  * These are entries which had been archived when they were in the "last" table,
 475  * and have in the meanwhile been updated. We have to:
 476  *    - Update prev_serial of their next object
 477  *    - Delete them!
 478  */
 479 
 480 tblObjList *find_unreferenced_history_entries(int date)
     /* [<][>][^][v][top][bottom][index][help] */
 481 {
 482 
 483   char query[MAXCMDLEN];
 484   SQ_result_set_t *result;
 485   SQ_row_t *row;
 486   int objid, seqid, serid;
 487 
 488   tblObjList *curListElmt = NULL;
 489   tblObjList *firstListElmt = NULL;
 490   tblObjList *tmpList = NULL;
 491 
 492   /* Find object_id, sequence_id of unreferenced history entries
 493    * This query returns all history entries which do not have a corresponding
 494    * (object_id, serial_id) in the serials table
 495    */
 496   /* XXX Bug! This will find (and then remove) objects that would be
 497    * needed in the future by deletions. */
 498 
 499   sprintf (query, "SELECT history.object_id, history.sequence_id, history.serial
 500                         FROM history LEFT JOIN serials
 501                         ON history.serial = serials.serial_id
 502                         WHERE serials.serial_id is NULL
 503                         AND history.serial != 0
 504                         AND history.timestamp < %d", date);
 505   execute_sql_query(query, &result);
 506 
 507   /* Foreach entry: */
 508   while ( (row = SQ_row_next(result)) != NULL )
 509     {
 510 
 511       /* Lock tables in writing... */
 512       /* XXX We don't need to do it, we are not deleting the objects here! */
 513       /* lock_last_history_serial_tables(); */
 514 
 515       /* XXX Error checking missing... */
 516       objid = atoi((const char *)row[0]);
 517       seqid = atoi((const char *)row[1]);
 518       serid = atoi((const char *)row[2]);
 519 
 520       /* Update prev_serial of the same object with next sequence_id */
 521       if (!update_prev_serial_of_next_object(objid, seqid, serid, "history"))
 522         { update_prev_serial_of_next_object(objid, seqid, serid, "last"); }
 523 
 524       /* Delete the entry */
 525       printf ("I am deleting this entry: %d, %d\n",objid, seqid);
 526 
 527       /* Don't delete the history entries directly! This will cause problems
 528          if the next serial is a deletion */
 529       /* Instead, add it in a list that will be batch-deleted at the end */
 530       /* PushTblObjList (objid, seqid, curListElmt); */
 531       
 532       tmpList = (tblObjList *)malloc(sizeof(tblObjList));
 533       tmpList->objid = objid;
 534       tmpList->seqid = seqid;
 535       tmpList->next = NULL;
 536 
 537       if (firstListElmt == NULL)
 538         {
 539           firstListElmt = tmpList;
 540           curListElmt = tmpList;
 541         }
 542       else
 543         {
 544           curListElmt->next = tmpList;
 545           curListElmt = curListElmt->next;
 546         }
 547 
 548       /* Unlock tables... */
 549       /* unlock_all_tables(); */
 550 
 551     }
 552 
 553   /* printf ("Elements to be deleted:\n");
 554      for (curListElmt = firstListElmt; curListElmt != NULL; curListElmt = curListElmt->next)
 555      {
 556      printf ("objid: %d, seqid: %d\n", curListElmt->objid, curListElmt->seqid);
 557      } */ 
 558 
 559   return (firstListElmt);
 560 
 561   return (0);
 562 
 563 } /* find_unreferenced_history_entries() */
 564 
 565 
 566 int delete_unreferenced_history_entries(tblObjList *objectsToDelete)
     /* [<][>][^][v][top][bottom][index][help] */
 567 {
 568 
 569   tblObjList *tmpObj;
 570 
 571   printf ("Elements to be deleted:\n");
 572   for (tmpObj = objectsToDelete; tmpObj != NULL; tmpObj = tmpObj->next)
 573     {
 574       printf ("objid: %d, seqid: %d\n", tmpObj->objid, tmpObj->seqid);
 575       delete_history_entry(tmpObj->objid, tmpObj->seqid);
 576     } 
 577 
 578   return(0);
 579 
 580 }
 581 
 582 
 583 /* PushTblObjList() */
 584 
 585 int PushTblObjList(int objid, int seqid, tblObjList *curListElmt)
     /* [<][>][^][v][top][bottom][index][help] */
 586 {
 587 
 588   tblObjList *tmpList;
 589 
 590   tmpList = (tblObjList *)malloc(sizeof(tblObjList));
 591   tmpList->objid = objid;
 592   tmpList->seqid = seqid;
 593   tmpList->next = NULL;
 594 
 595   if (curListElmt == NULL)
 596     {
 597       curListElmt = tmpList;
 598     }
 599   else
 600     {
 601       curListElmt->next = tmpList;
 602       /* curListElmt = tmpList; */
 603     }
 604 
 605   printf ("Inside PushTblObjList: %d, %d\n", curListElmt->objid, curListElmt->seqid);
 606 
 607   return(0);
 608 
 609 } /* PushTblObjList() */
 610 
 611 
 612 /* get_highest_serial_before_date()
 613  * We get the biggest serial for which the history or last timestamp is lower than
 614  * the defined timestamp 
 615  */
 616 
 617 int get_highest_serial_before_date (int *highest_serial_ptr, int date)
     /* [<][>][^][v][top][bottom][index][help] */
 618 {
 619 
 620   char query[MAXCMDLEN];
 621   SQ_result_set_t *result;
 622   SQ_row_t *row;
 623 
 624   /* sprintf (query, "SELECT MAX(serials.serial_id) FROM history,serials 
 625                         WHERE history.timestamp < %d 
 626                         AND history.object_id = serials.object_id 
 627                         AND history.sequence_id = serials.sequence_id ", date); */
 628 
 629   sprintf (query, "SELECT MAX(serials.serial_id) 
 630                         FROM serials NATURAL LEFT JOIN last NATURAL LEFT JOIN history
 631                         WHERE ((last.timestamp < %d 
 632                                 AND last.object_id = serials.object_id 
 633                                 AND last.sequence_id = last.sequence_id)
 634                         OR (history.timestamp < %d 
 635                                 AND history.object_id = serials.object_id 
 636                                 AND history.sequence_id = serials.sequence_id))",
 637            date, date);
 638 
 639   execute_sql_query(query, &result);
 640   if ( (row = SQ_row_next(result)) != NULL )
 641     {
 642       *highest_serial_ptr = row[0] ? atoi((const char *)row[0]) : 0;
 643       /* printf ("Highest serial ID: %d\n", *highest_serial_ptr); */
 644     }
 645 
 646   SQ_free_result(result);
 647 
 648   return(0);
 649 
 650 } /* get_highest_serial_before_date() */
 651 
 652 
 653 
 654 
 655 
 656 /* archive_serials_and_history():
 657  * This function contains the core algorithm that manipulates the last,
 658  * history and serials tables and archives them into serials_archive
 659  * and history_archive tables.
 660  */
 661 
 662 int archive_serials_and_history (int highest_serial)
     /* [<][>][^][v][top][bottom][index][help] */
 663 {
 664 
 665   char query[MAXCMDLEN];
 666   SQ_result_set_t *result;
 667   SQ_row_t *row;
 668   int serial, atlast, objid, seqid, op;
 669   char *tablename;
 670   int timestamp;
 671   int rc;
 672 
 673 
 674   /* XXX Missing: crash recovery handling. */
 675 
 676 
 677   /* Get the entries for each serial */
 678   /* One word about the "<": I had "<=" but if highest_serial
 679      is the CURRENTSERIAL, it causes big problems to UD! 
 680      (at least in mirror mode...) */
 681   sprintf (query, "SELECT serials.serial_id, serials.atlast, 
 682                         ELT(serials.atlast+1,'history','last','failed_transaction'), 
 683                         serials.object_id, serials.sequence_id, serials.operation 
 684                         FROM serials 
 685                         WHERE serials.serial_id < %d
 686                         ORDER BY serials.serial_id", highest_serial);
 687   execute_sql_query(query, &result);
 688 
 689   /* Loop on every serial */
 690   while ( (row = SQ_row_next(result)) != NULL )
 691     {
 692 
 693       /* The lock is inserted here, inside the loop, because it is
 694        * a write lock, which disallows the reading of the table.
 695        * Since one concerned table is "last", the queries would
 696        * be blocked.
 697        * By freeing the lock at the end of every loop, we are assured
 698        * that the reads in queue are executed before a new lock is set.
 699        */
 700 
 701       /* Lock (write lock!) relevant tables */
 702       if ((rc = lock_last_history_serial_tables()) != 0)
 703         { return rc; }
 704 
 705       /* XXX Add stronger error checking: NULL rows should never happen */
 706       serial = row[0] ? atoi((const char *)row[0]) : 0;
 707       atlast = row[1] ? atoi((const char *)row[1]) : IN_UNDEF_TABLE;
 708 
 709       if (row[2] == NULL)
 710         {
 711           /* That should never happen! */
 712           fprintf (stderr, "Fatal: No pointer to table\n");
 713           return (-1);
 714         }
 715       else
 716         {
 717           tablename = strdup((const char *)row[2]);
 718         }
 719 
 720       objid = atoi((const char *)row[3]);
 721       seqid = atoi((const char *)row[4]);
 722       op = atoi((const char *)row[5]);
 723 
 724       /* printf ("Serial: %d; Atlast: %d; Objid: %d; Seqid: %d; Op: %d; Tablename: %s\n",serial, atlast, objid, seqid, op, tablename); */
 725 
 726       update_hs_auxiliary_checkpoint(serial, objid, seqid, 0, CHKP_NOOP);
 727 
 728       if (atlast == IN_FAILED_TRANSACTION_TABLE)
 729         {
 730 
 731           /* The serial points to a failed transaction */
 732 
 733           /* Archive serial */
 734           archive_serial(serial, objid, seqid, op);
 735           update_hs_auxiliary_checkpoint(serial, objid, seqid, 0, CHKP_ARCHIVED_SERIAL);
 736 
 737           /* Archive failed transaction */
 738           archive_failed_transaction(serial);
 739 
 740           /* Delete serial */
 741           delete_serial_entry(serial);
 742 
 743           /* Delete failed transaction */
 744           delete_failed_transaction_entry(serial);
 745 
 746 
 747         }
 748       else /* atlast == (IN_LAST_TABLE || IN_HISTORY_TABLE) */
 749         {
 750 
 751           if (op == OP_DELETE)
 752             {
 753 
 754               /* Then it must be in the history */
 755 
 756               if (debug) printf ("Deleted serial. Objid: %d, seqid: %d, serial: %d\n",objid, seqid, serial);
 757 
 758               /* We need to update the prev_serial of the next element, if there is one...
 759                * This compensates for UPD = DEL + ADD; the ADD is treated with the same
 760                * object_id and sequence_id++ */
 761               if (!update_prev_serial_of_next_object(objid, seqid, serial, "history")) 
 762                 update_prev_serial_of_next_object(objid, seqid, serial, "last");
 763 
 764               /* Archive serial */
 765               archive_serial(serial, objid, seqid, op);
 766 
 767               /* XXX Fixme: no timestamp is archived if this DEL is part of a DEL+ADD .
 768                * This could be solved by fetching the timestamp from the next
 769                * sequence_id (which is the corresponding UPD), if existent.
 770                */
 771 
 772               /* Fetch timestamp from the corresponding empty last entry */
 773               timestamp = fetch_timestamp_from_last(objid, seqid);
 774 
 775               /* printf ("Timestamp for serial %d: %d\n",serial, timestamp); */
 776 
 777               /* Archive history:
 778                * we need a special function here because we need to archive
 779                * history.serial as history_archive.prev_serial .
 780                */
 781               copy_deleted_object_into_history_archive(serial, objid, seqid, "history");
 782 
 783               /* Update history archive with correct timestamp */
 784               /* XXX We don't really need a function which also updates the seq_id */
 785               update_history_sequence_id_and_timestamp(serial, seqid, timestamp);
 786 
 787 
 788               /* Delete serial */
 789               delete_serial_entry(serial);
 790               
 791               /* Delete corresponding empty last entry: it has a seq_id of 0 */
 792               /* XXX It must only do so if the entry to be deleted is not the
 793                  highest object_id */
 794               /* To be fixed */
 795               /* delete_last_entry(objid, 0); */
 796 
 797               /* Delete history entry */
 798               delete_history_entry(objid, seqid);
 799 
 800 
 801             }
 802           else /* It is an update */
 803             {
 804 
 805               if (atlast == IN_LAST_TABLE )
 806                 {
 807 
 808                   /* Archive serial */
 809                   archive_serial(serial, objid, seqid, op);
 810 
 811                   /* Archive last */
 812                   archive_last(serial, objid, seqid);
 813 
 814                   /* Update serial element of the entry in last table */
 815                   update_serial_in_last(objid, seqid, serial);
 816 
 817                   /* Delete serial */
 818                   delete_serial_entry(serial);
 819 
 820                   /* !!!Do not delete the "last" entry!!! */
 821 
 822                 }
 823               else /* atlast == IN_HISTORY_TABLE */
 824                 {
 825 
 826                   /* We check for the next object, in order to update
 827                    * its prev_serial. We first look in the history table,
 828                    * then in the last table, otherwise there is no such object
 829                    * => the following update is in fact a deletion...
 830                    */
 831 
 832                   if (check_if_next_is_deletion == 0)
 833                     {
 834 
 835                       /* update_prev_serial_of_next_object() returns the number of
 836                        * affected rows: this shows us if the operation has been successful
 837                        * or not */
 838                       if (!update_prev_serial_of_next_object(objid, seqid, serial, "history")) 
 839                         update_prev_serial_of_next_object(objid, seqid, serial, "last");
 840 
 841                       /* Archive serial */
 842                       archive_serial(serial, objid, seqid, op);
 843 
 844                       /* Archive history */
 845                       archive_history(serial, objid, seqid);
 846 
 847                       /* Delete serial */
 848                       delete_serial_entry(serial);
 849 
 850                       /* Delete history */
 851                       delete_history_entry(objid, seqid);
 852                       
 853                     }
 854                   else
 855                     {
 856 
 857                       /* Archive serial */
 858                       archive_serial(serial, objid, seqid, op);
 859 
 860                       /* Archive history */
 861                       archive_history(serial, objid, seqid);
 862 
 863                       /* Update serial in -current- history entry */
 864                       update_serial_in_history(objid, seqid, serial);
 865 
 866                       /* Delete serial */
 867                       delete_serial_entry(serial);
 868 
 869                       /* Do not delete current history entry! It will be needed
 870                          by the deleting serial */
 871 
 872                     }
 873 
 874                 }
 875 
 876             }
 877 
 878         }
 879 
 880       /* Unlock relevant tables */
 881       unlock_all_tables();
 882 
 883     }
 884 
 885   SQ_free_result(result);
 886 
 887 
 888 
 889   return(0);
 890 
 891 } /* archive_serials_and_history() */
 892 
 893 
 894 
 895 
 896 
 897 
 898 int lock_last_history_serial_tables()
     /* [<][>][^][v][top][bottom][index][help] */
 899 {
 900 
 901   char cmd[MAXCMDLEN];
 902 
 903   /* No real choice - we must lock the tables in write mode */
 904 
 905   sprintf (cmd, "LOCK TABLES last WRITE, history WRITE, failed_transaction WRITE, serials WRITE, serials_archive WRITE, history_archive WRITE, hs_auxiliary WRITE");
 906 
 907   return (execute_sql_command(cmd));
 908 
 909 } /* lock_last_history_serial_tables() */
 910 
 911 
 912 
 913 int unlock_all_tables()
     /* [<][>][^][v][top][bottom][index][help] */
 914 {
 915 
 916   char cmd[MAXCMDLEN];
 917 
 918   sprintf (cmd, "UNLOCK TABLES");
 919 
 920   return (execute_sql_command(cmd));
 921 
 922 } /* unlock_all_tables() */
 923 
 924 
 925 int update_hs_auxiliary_checkpoint(int ser_id, int obj_id, int seq_id, int timestamp, int checkpoint)
     /* [<][>][^][v][top][bottom][index][help] */
 926 {
 927 
 928   char cmd[MAXCMDLEN];
 929 
 930   sprintf (cmd,"UPDATE hs_auxiliary 
 931                 SET serial = %d,
 932                 object_id = %d,
 933                 sequence_id = %d,
 934                 timestamp = %d,
 935                 checkpoint = %d",
 936            ser_id, obj_id, seq_id, timestamp, checkpoint);
 937 
 938   return (execute_sql_command(cmd));
 939 
 940 } /* update_hs_auxiliary_checkpoint() */
 941 
 942 
 943 int archive_serial (int ser_id, int obj_id, int seq_id, int op)
     /* [<][>][^][v][top][bottom][index][help] */
 944 {
 945 
 946   /* Put the given values into the serials_archive table */
 947 
 948   char cmd[MAXCMDLEN];
 949 
 950   sprintf (cmd, "INSERT INTO serials_archive (serial_id, object_id, sequence_id, operation)
 951                  VALUES (%d, %d, %d, %d)",
 952            ser_id, obj_id, seq_id, op);
 953 
 954   return (execute_sql_command(cmd));
 955 
 956 } /* archive_serial() */
 957 
 958 
 959 
 960 
 961 int archive_failed_transaction (int ser_id)
     /* [<][>][^][v][top][bottom][index][help] */
 962 {
 963 
 964   char cmd[MAXCMDLEN];
 965   
 966   sprintf (cmd,"INSERT INTO history_archive (serial, timestamp, object)
 967                 SELECT failed_transaction.serial_id, failed_transaction.timestamp, failed_transaction.object
 968                 FROM failed_transaction
 969                 WHERE failed_transaction.serial_id = %d",
 970            ser_id);
 971 
 972   return (execute_sql_command(cmd));
 973 
 974 } /* archive_failed_transaction() */
 975 
 976 
 977 int copy_into_history_archive (int ser_id, int obj_id, int seq_id, const char *tablename)
     /* [<][>][^][v][top][bottom][index][help] */
 978 {
 979 
 980   char cmd[MAXCMDLEN];
 981   
 982   sprintf (cmd,"INSERT INTO history_archive (object_id, sequence_id, serial, prev_serial, timestamp, object_type, object)
 983                 SELECT serials.object_id, serials.sequence_id, serials.serial_id, %s.prev_serial, %s.timestamp, %s.object_type, %s.object
 984                 FROM serials,%s 
 985                 WHERE serials.serial_id = %d 
 986                 AND %s.object_id = %d 
 987                 AND %s.sequence_id = %d",
 988            tablename, tablename, tablename, tablename, tablename, ser_id, tablename, obj_id, tablename, seq_id);
 989 
 990   return (execute_sql_command(cmd));
 991 
 992 } /* copy_into_history_archive() */
 993 
 994 
 995 
 996 int copy_deleted_object_into_history_archive (int ser_id, int obj_id, int seq_id, const char *tablename)
     /* [<][>][^][v][top][bottom][index][help] */
 997 {
 998 
 999   /* The difference here is that we archive the history.serial as history_archive.prev_serial .
1000    * This is only needed for history objects corresponding to OP_DELETE,
1001    * where the row actually corresponds to the last update before the deletion. */
1002 
1003   char cmd[MAXCMDLEN];
1004   int affected_rows;
1005 
1006   sprintf (cmd,"INSERT INTO history_archive (object_id, sequence_id, serial, prev_serial, timestamp, object_type, object)
1007                 SELECT serials.object_id, serials.sequence_id, serials.serial_id, %s.serial, %s.timestamp, %s.object_type, %s.object
1008                 FROM serials,%s 
1009                 WHERE serials.serial_id = %d 
1010                 AND %s.object_id = %d 
1011                 AND %s.sequence_id = %d",
1012            tablename, tablename, tablename, tablename, tablename, ser_id, tablename, obj_id, tablename, seq_id);
1013 
1014   affected_rows = execute_sql_command(cmd);
1015   if (debug) printf ("copy_deleted_object_into_history_archive (%d, %d, %d, %s): affected rows %d\n",ser_id,obj_id,seq_id,tablename,affected_rows);
1016   return (affected_rows);
1017 
1018 } /* copy_deleted_object_into_history_archive() */
1019 
1020 
1021 
1022 int update_history_sequence_id_and_timestamp (int ser_id, int new_seq_id, int new_timestamp)
     /* [<][>][^][v][top][bottom][index][help] */
1023 {
1024 
1025   char cmd[MAXCMDLEN];
1026 
1027   sprintf (cmd,"UPDATE history_archive
1028                 SET timestamp = %d, sequence_id = %d
1029                 WHERE serial = %d",
1030            new_timestamp, new_seq_id, ser_id);
1031 
1032   return (execute_sql_command(cmd));
1033 
1034 } /* update_history_sequence_id_and_timestamp() */
1035 
1036 
1037 
1038 /* check_if_next_is_deletion()
1039  * This functions checks if there is a row in the serials
1040  * table with same obj_id and seq_id, but a delete operation.
1041  * This would mean that we are dealing with a last-update-before-deletion.
1042  */
1043 
1044 int check_if_next_is_deletion (int obj_id, int seq_id)
     /* [<][>][^][v][top][bottom][index][help] */
1045 {
1046 
1047   char query[MAXCMDLEN];
1048   SQ_result_set_t *result;
1049   SQ_row_t *row;
1050   int serial = 0;
1051 
1052   sprintf (query, "SELECT serial_id, atlast FROM serials 
1053                         WHERE object_id = %d AND sequence_id = %d AND operation = %d", 
1054            obj_id, seq_id, OP_DELETE);
1055 
1056   execute_sql_query(query, &result);
1057   if ( (row = SQ_row_next(result)) != NULL)
1058     serial = atoi((const char *)row[0]);
1059 
1060   SQ_free_result(result);
1061 
1062   return(serial);
1063 
1064 } /* check_if_next_is_deletion() */
1065 
1066 
1067 
1068 int update_prev_serial_of_object (int obj_id, int seq_id, int prev_ser, const char *tablename)
     /* [<][>][^][v][top][bottom][index][help] */
1069 {
1070 
1071   char cmd[MAXCMDLEN];
1072 
1073   sprintf (cmd,"UPDATE %s 
1074                 SET prev_serial = %d 
1075                 WHERE object_id = %d
1076                 AND sequence_id = %d",
1077            tablename, prev_ser, obj_id, seq_id);
1078 
1079   return(execute_sql_command(cmd));
1080 
1081 } /* update_prev_serial_of_object() */
1082 
1083 
1084 
1085 int update_serial_of_object (int obj_id, int seq_id, int ser_id, const char *tablename)
     /* [<][>][^][v][top][bottom][index][help] */
1086 {
1087 
1088   char cmd[MAXCMDLEN];
1089 
1090   sprintf (cmd,"UPDATE %s 
1091                 SET serial = %d 
1092                 WHERE object_id = %d
1093                 AND sequence_id = %d",
1094            tablename, ser_id, obj_id, seq_id);
1095 
1096   return(execute_sql_command(cmd));
1097 
1098 } /* update_serial_of_object() */
1099 
1100 
1101 
1102 int delete_serial_entry (int ser_id)
     /* [<][>][^][v][top][bottom][index][help] */
1103 {
1104 
1105   char cmd[MAXCMDLEN];
1106 
1107   sprintf (cmd, "DELETE FROM serials WHERE serial_id = %d", ser_id);
1108 
1109   return (execute_sql_command(cmd));
1110 
1111 } /* delete_serial_entry() */
1112 
1113 
1114 int delete_failed_transaction_entry (int ser_id)
     /* [<][>][^][v][top][bottom][index][help] */
1115 {
1116 
1117   char cmd[MAXCMDLEN];
1118 
1119   sprintf (cmd, "DELETE FROM failed_transaction WHERE serial_id = %d",ser_id);
1120 
1121   return (execute_sql_command(cmd));
1122 
1123 } /* delete_failed_transaction_entry() */
1124 
1125 
1126 
1127 int delete_entry_from_object_table (int obj_id, int seq_id, const char* tablename)
     /* [<][>][^][v][top][bottom][index][help] */
1128 {
1129 
1130   char cmd[MAXCMDLEN];
1131   int affected_rows;
1132 
1133   if (debug) printf ("Deleting %s entry. Objid: %d, seqid: %d\n",tablename, obj_id, seq_id);
1134 
1135   sprintf (cmd, "DELETE FROM %s WHERE object_id = %d AND sequence_id = %d",
1136            tablename, obj_id, seq_id);
1137 
1138   affected_rows = execute_sql_command(cmd);
1139   if (debug) printf ("delete_entry_from_object_table (%d, %d, %s): affected rows %d\n",obj_id,seq_id,tablename,affected_rows);
1140   return (affected_rows);
1141 
1142 } /* delete_entry_from_object_table() */
1143 
1144 
1145 int fetch_timestamp_from_last (int obj_id, int seq_id)
     /* [<][>][^][v][top][bottom][index][help] */
1146 {
1147 
1148   int timestamp = 0;
1149   char query[MAXCMDLEN];
1150   SQ_result_set_t *result;
1151   SQ_row_t *row;
1152 
1153   sprintf (query, "SELECT timestamp FROM last WHERE object_id = %d AND sequence_id = %d", 
1154            obj_id, seq_id);
1155 
1156   execute_sql_query(query, &result);
1157   if ( (row = SQ_row_next(result)) != NULL)
1158     timestamp = atoi((const char *)row[0]);
1159 
1160   SQ_free_result(result);
1161 
1162   return(timestamp);
1163 
1164 }
1165 
1166 
1167 int optimize_sql_table(const char* tablename)
     /* [<][>][^][v][top][bottom][index][help] */
1168 {
1169 
1170   char cmd[MAXCMDLEN];
1171 
1172   sprintf (cmd, "OPTIMIZE TABLE %s", tablename);
1173 
1174   return (execute_sql_command(cmd));
1175 
1176 } /* optimize_sql_table() */
1177 
1178 
1179 int execute_sql_query (const char *query, SQ_result_set_t **result_ptr)
     /* [<][>][^][v][top][bottom][index][help] */
1180 {
1181 
1182   int state;
1183 
1184   state = SQ_execute_query(connection, query, result_ptr);
1185   if (state != 0)
1186     {
1187       fprintf (stderr, "Fatal:\n Offending query: %s\n Error: %s\n",query,SQ_error(connection));
1188       die;
1189     }
1190 
1191   return(state);
1192 
1193 }
1194 
1195 
1196 
1197 int execute_sql_command (const char *cmd)
     /* [<][>][^][v][top][bottom][index][help] */
1198 {
1199 
1200   int state;
1201 
1202   state = SQ_execute_query(connection, cmd, NULL);
1203   if (state != 0)
1204     {
1205       fprintf (stderr, "Fatal:\n Offending command: %s\n Error: %s\n",cmd,SQ_error(connection));
1206       die;
1207     }
1208 
1209   return(SQ_get_affected_rows(connection));
1210 
1211 }
1212 
1213 
1214 
1215 
1216 void usage(char *argv[])
     /* [<][>][^][v][top][bottom][index][help] */
1217 {
1218 
1219   printf ("Usage: \n\n");
1220   printf ("  %s [-?] [-h host] [-P port] [-u user] [-p password] [-d database]\n", argv[0]);
1221   printf ("     [-T date|-S seconds|-M minutes|-H hours|-D days|-W weeks] \n");
1222 
1223   printf ("\nGeneral options:\n");
1224   printf ("   -?: This text\n");
1225 
1226   printf ("\nSQL Options:\n");
1227   printf ("   -h: host \t\t(default: %s)\n",MYSQL_HOST);
1228   printf ("   -P: port \t\t(default: %d)\n",MYSQL_PORT);
1229   printf ("   -u: user \t\t(default: %s)\n",MYSQL_USER);
1230   printf ("   -p: password \t(default: %s)\n",MYSQL_PSWD);
1231   printf ("   -d: database name \t(default: %s)\n",MYSQL_DB);
1232 
1233   printf ("\nTime-related options: (one and only one must be specified)\n");
1234   printf ("   -T date: Date before which to archive (secs from the Epoch)\n");
1235   printf ("   -S seconds: Seconds elapsed between the date to archive and now\n");
1236   printf ("   -M minutes: Minutes elapsed between the date to archive and now\n");
1237   printf ("   -H hours: Hours elapsed between the date to archive and now\n");
1238   printf ("   -D days: Days elapsed between the date to archive and now\n");
1239   printf ("   -W weeks: Weeks elapsed between the date to archive and now\n");
1240   exit(1);
1241 
1242 }
1243 
1244 

/* [<][>][^][v][top][bottom][index][help] */