/***************************************************************************** * RRDtool 1.7.2 Copyright by Tobi Oetiker * Copyright by Florian Forster, 2008 ***************************************************************************** * rrd_update.c RRD Update Function *****************************************************************************/ #include "rrd_tool.h" #if defined(_WIN32) && !defined(__CYGWIN__) && !defined(__CYGWIN32__) #include #include #include #endif #include #ifdef HAVE_STDINT_H # include #endif #include "rrd_hw.h" #include "rrd_rpncalc.h" #include "rrd_is_thread_safe.h" #include "unused.h" #include "rrd_client.h" #include "rrd_update.h" #include #include "rrd_strtod.h" #if defined(_WIN32) && !defined(__CYGWIN__) && !defined(__CYGWIN32__) && !defined(__MINGW32__) /* Remark: HAVE_GETTIMEOFDAY could be used here alternatively */ /* * WIN32 does not have gettimeofday and struct timeval. This is a quick and dirty * replacement. */ #include struct __timezone { int tz_minuteswest; /* minutes W of Greenwich */ int tz_dsttime; /* type of dst correction */ }; static int gettimeofday( struct timeval *t, struct __timezone *tz) { struct _timeb current_time; _ftime(¤t_time); t->tv_sec = current_time.time; t->tv_usec = current_time.millitm * 1000; return 0; } #endif /* FUNCTION PROTOTYPES */ static int _rrd_updatex( const char *filename, const char *tmplt, int extra_flags, int argc, const char **argv, rrd_info_t *); static int allocate_data_structures( rrd_t *rrd, char ***updvals, rrd_value_t **pdp_temp, const char *tmplt, long **tmpl_idx, unsigned long *tmpl_cnt, unsigned long **rra_step_cnt, unsigned long **skip_update, rrd_value_t **pdp_new); static int parse_template( rrd_t *rrd, const char *tmplt, unsigned long *tmpl_cnt, long *tmpl_idx); static int process_arg( char *step_start, rrd_t *rrd, rrd_file_t *rrd_file, unsigned long rra_begin, time_t *current_time, unsigned long *current_time_usec, rrd_value_t *pdp_temp, rrd_value_t *pdp_new, unsigned long *rra_step_cnt, char **updvals, long *tmpl_idx, unsigned long tmpl_cnt, rrd_info_t ** pcdp_summary, int version, unsigned long *skip_update, int *schedule_smooth); static int parse_ds( rrd_t *rrd, char **updvals, long *tmpl_idx, char *input, unsigned long tmpl_cnt, time_t *current_time, unsigned long *current_time_usec, int version); static int get_time_from_reading( rrd_t *rrd, char timesyntax, char **updvals, time_t *current_time, unsigned long *current_time_usec, int version); static int update_pdp_prep( rrd_t *rrd, char **updvals, rrd_value_t *pdp_new, double interval); static int calculate_elapsed_steps( rrd_t *rrd, unsigned long current_time, unsigned long current_time_usec, double interval, double *pre_int, double *post_int, unsigned long *proc_pdp_cnt); static void simple_update( rrd_t *rrd, double interval, rrd_value_t *pdp_new); static int process_all_pdp_st( rrd_t *rrd, double interval, double pre_int, double post_int, unsigned long elapsed_pdp_st, rrd_value_t *pdp_new, rrd_value_t *pdp_temp); static int process_pdp_st( rrd_t *rrd, unsigned long ds_idx, double interval, double pre_int, double post_int, long diff_pdp_st, rrd_value_t *pdp_new, rrd_value_t *pdp_temp); static int update_all_cdp_prep( rrd_t *rrd, unsigned long *rra_step_cnt, unsigned long rra_begin, rrd_file_t *rrd_file, unsigned long elapsed_pdp_st, unsigned long proc_pdp_cnt, rrd_value_t **last_seasonal_coef, rrd_value_t **seasonal_coef, rrd_value_t *pdp_temp, unsigned long *skip_update, int *schedule_smooth); static int do_schedule_smooth( rrd_t *rrd, unsigned long rra_idx, unsigned long elapsed_pdp_st); static int update_cdp_prep( rrd_t *rrd, unsigned long elapsed_pdp_st, unsigned long start_pdp_offset, unsigned long *rra_step_cnt, int rra_idx, rrd_value_t *pdp_temp, rrd_value_t *last_seasonal_coef, rrd_value_t *seasonal_coef, int current_cf); static void initialize_cdp_val( unival *scratch, int current_cf, rrd_value_t pdp_temp_val, unsigned long start_pdp_offset, unsigned long pdp_cnt); static void reset_cdp( rrd_t *rrd, unsigned long elapsed_pdp_st, rrd_value_t *pdp_temp, rrd_value_t *last_seasonal_coef, rrd_value_t *seasonal_coef, int rra_idx, int ds_idx, int cdp_idx, enum cf_en current_cf); static rrd_value_t initialize_carry_over( rrd_value_t pdp_temp_val, int current_cf, unsigned long elapsed_pdp_st, unsigned long start_pdp_offset, unsigned long pdp_cnt); static rrd_value_t calculate_cdp_val( rrd_value_t cdp_val, rrd_value_t pdp_temp_val, unsigned long elapsed_pdp_st, int current_cf, int i, int ii); static int update_aberrant_cdps( rrd_t *rrd, rrd_file_t *rrd_file, unsigned long rra_begin, unsigned long elapsed_pdp_st, rrd_value_t *pdp_temp, rrd_value_t **seasonal_coef); static int write_to_rras( rrd_t *rrd, rrd_file_t *rrd_file, unsigned long *rra_step_cnt, unsigned long rra_begin, time_t current_time, unsigned long *skip_update, rrd_info_t ** pcdp_summary); static int write_RRA_row( rrd_file_t *rrd_file, rrd_t *rrd, unsigned long rra_idx, unsigned short CDP_scratch_idx, rrd_info_t ** pcdp_summary, time_t rra_time); static int smooth_all_rras( rrd_t *rrd, rrd_file_t *rrd_file, unsigned long rra_begin); #if !defined(HAVE_MMAP) || defined(HAVE_LIBRADOS) static int write_changes_to_disk( rrd_t *rrd, rrd_file_t *rrd_file, int version); #endif /* * normalize time as returned by gettimeofday. usec part must * be always >= 0 */ static void normalize_time( struct timeval *t) { if (t->tv_usec < 0) { t->tv_sec--; t->tv_usec += 1e6L; } } /* * Sets current_time and current_time_usec based on the current time. * current_time_usec is set to 0 if the version number is 1 or 2. */ static void initialize_time( time_t *current_time, unsigned long *current_time_usec, int version) { struct timeval tmp_time; /* used for time conversion */ gettimeofday(&tmp_time, 0); normalize_time(&tmp_time); *current_time = tmp_time.tv_sec; if (version >= 3) { *current_time_usec = tmp_time.tv_usec; } else { *current_time_usec = 0; } } #define IFDNAN(X,Y) (isnan(X) ? (Y) : (X)); rrd_info_t *rrd_update_v( int argc, char **argv) { struct optparse_long longopts[] = { {"template", 't', OPTPARSE_REQUIRED}, {"skip-past-updates", 's', OPTPARSE_NONE}, {0}, }; struct optparse options; int opt; const char *tmplt = NULL; int extra_flags = 0; rrd_info_t *result = NULL; rrd_infoval_t rc; char *opt_daemon = NULL; rc.u_int = -1; optparse_init(&options, argc, argv); while ((opt = optparse_long(&options, longopts, NULL)) != -1) { switch (opt) { case 't': tmplt = options.optarg; break; case 's': extra_flags |= RRD_SKIP_PAST_UPDATES; break; case '?': rrd_set_error("%s", options.errmsg); goto end_tag; } } opt_daemon = getenv (ENV_RRDCACHED_ADDRESS); if (opt_daemon != NULL && ! strcmp(opt_daemon,"")) { rrd_set_error ("The \"%s\" environment variable is defined, " "but \"%s\" cannot work with rrdcached. Either unset " "the environment variable or use \"update\" instead.", ENV_RRDCACHED_ADDRESS, options.argv[0]); goto end_tag; } /* need at least 2 arguments: filename, data. */ if (options.argc - options.optind < 2) { rrd_set_error("Not enough arguments"); goto end_tag; } rc.u_int = 0; result = rrd_info_push(NULL, sprintf_alloc("return_value"), RD_I_INT, rc); rc.u_int = _rrd_updatex(options.argv[options.optind], tmplt,extra_flags, options.argc - options.optind - 1, (const char **) (options.argv + options.optind + 1), result); result->value.u_int = rc.u_int; end_tag: return result; } static char *rrd_get_file_template(const char *filename) /* {{{ */ { rrd_t rrd; rrd_file_t *rrd_file; unsigned int i; size_t len=0; char *filetemplate = NULL; //template is a key word in C++ , see http://msdn.microsoft.com/de-de/library/2e6a4at9.aspx /* open file */ rrd_init(&rrd); rrd_file = rrd_open(filename, &rrd, RRD_READONLY | RRD_LOCK); if (rrd_file == NULL) goto err_free; /* read the data from the rrd */ for (i = 0; i < rrd.stat_head->ds_cnt; i++) { len += strlen(rrd.ds_def[i].ds_nam)+1; } /* now that we got it allocate memory */ filetemplate = (char *) malloc(len); if (!filetemplate) goto err_close; filetemplate[0] = 0; /* fill in for real */ for (i = 0; i < rrd.stat_head->ds_cnt; i++) { if (i) strcat(filetemplate,":"); strcat(filetemplate,rrd.ds_def[i].ds_nam); } err_close: rrd_close(rrd_file); err_free: rrd_free(&rrd); return (filetemplate); } /* }}} const char *rrd_get_file_template */ /* the file-template-cache implementation */ static GTree *rrd_file_template_cache = NULL; /* the necessary functions for the gtree */ static gint cache_compare_names (gconstpointer name1, gconstpointer name2, gpointer data) { (void)(data); /* to avoid unused message */ return (strcmp((const char *)name1, (const char *)name2)); } static void cache_destroy(gpointer data) { free(data); } static const char *rrd_get_file_template_format(const char *filename) /* {{{ */ { char *format = NULL; /* create rrd_file_template_cache if needed */ if (!rrd_file_template_cache) { rrd_file_template_cache = g_tree_new_full ( cache_compare_names, NULL, cache_destroy, cache_destroy); if (!rrd_file_template_cache) return NULL; } /* fetch from cache */ format = (char *) g_tree_lookup(rrd_file_template_cache, filename); if (format) return format; /* fetch information from file */ format = rrd_get_file_template(filename); if (!format) return NULL; /* create copy of filename */ filename = strdup(filename); if (!filename) goto free_format; /* and add object to tree */ g_tree_insert (rrd_file_template_cache, (char *)filename, format); return format; free_format: free((void *)format); return NULL; } /* }}} const char *rrd_get_file_template_format */ static size_t _count_fields(const char *field) { int c=1; /* check for empty */ if (!field) return 0; if (!field[0]) return 0; /* now count */ while((field = strchr(field, ':'))) { c++; field++; } return c; } static int _concat_field_n(char* string, const char *value, int field) { const char *colon; size_t len; /* find the field */ while (field) { value = strchr(value, ':'); if (!value) return -1; value++; field--; } /* get the end of string and calculate length */ colon = strchr(value, ':'); if (colon) { len = colon-value; } else { len = strlen(value); } /* now concat strings */ strncat(string, value, len); /* return 1 field */ return 1; } static int _concat_field( char *string, const char* tpl, const char* value, const char* field) { int fieldidx=0; size_t len = 0; /* get length */ char *colon = strchr((char *)field, ':'); if (colon) { len=colon-field; } else len=strlen(field); /* concat ":" */ strcat(string, ":"); /* find field in tpl */ while(tpl) { /* check if it matches (including terminating 0 or :) */ if((strncmp(tpl, field, len)==0)) { if ((tpl[len] == 0) || (tpl[len] == ':')) { return _concat_field_n( string, value, fieldidx+1); } } /* increment tpl */ tpl = strchr(tpl, ':'); if (tpl) tpl++; /* and also increase the field index */ fieldidx++; } /* concat "U" */ strcat(string, "U"); /* and return "not found" */ return 0; } static char *rrd_map_template_to_values(const char *tpl, /* {{{ */ const char *file_tpl, const char *value) { char *mapped = NULL; size_t fields_count = 0; size_t fields_tpl = _count_fields(tpl); size_t fields_file_tpl = _count_fields(file_tpl); size_t fields_value = _count_fields(value); const char *ptr; size_t len; size_t i; int ret; /* check number of fields*/ if (fields_tpl != fields_value-1) { rrd_set_error("rrd_map_template_to_values: " "mismatch of number of fields in template (%zu) " "with number of fields in values (%zu)", fields_tpl,fields_value-1); return NULL; } if (fields_tpl > fields_file_tpl) { rrd_set_error("rrd_map_template_to_values: " "number of fields in template (%zu) " "bigger than number of fields in rrdfile (%zu)", fields_tpl,fields_file_tpl); return NULL; } /* now calculate effective length and allocate it */ len = strlen(value) /* length of the value */ + 1 /* terminating null byte */ + (fields_file_tpl-fields_tpl) /* number of fields that we have * more in the file_template * compared to the given template */ * 2 /* = strlen(":U") */; mapped = (char *) malloc(len); if (!mapped) return NULL; mapped[0] = 0; /* first copy timestamp */ if (_concat_field_n(mapped,value,0)<0) goto err; /* now walk the list of fields from rrdtemplate*/ ptr = file_tpl; for (i=0; i 0) if (!rrd_test_error()) rrd_set_error("Failed sending the values to rrdcached: %s", rrd_strerror (rc)); } out: if (tmplt != NULL) { free(tmplt); tmplt = NULL; } if (opt_daemon != NULL) { free (opt_daemon); opt_daemon = NULL; } return rc; } int rrd_update_r( const char *filename, const char *tmplt, int argc, const char **argv) { return _rrd_updatex(filename, tmplt, 0, argc, argv, NULL); } int rrd_update_v_r( const char *filename, const char *tmplt, int argc, const char **argv, rrd_info_t * pcdp_summary) { return _rrd_updatex(filename, tmplt, 0, argc, argv, pcdp_summary); } int rrd_updatex_r( const char *filename, const char *tmplt, int extra_flags, int argc, const char **argv) { return _rrd_updatex(filename, tmplt, extra_flags, argc, argv, NULL); } int rrd_updatex_v_r( const char *filename, const char *tmplt, int extra_flags, int argc, const char **argv, rrd_info_t * pcdp_summary) { return _rrd_updatex(filename, tmplt, extra_flags, argc, argv, pcdp_summary); } static int _rrd_updatex( const char *filename, const char *tmplt, int extra_flags, int argc, const char **argv, rrd_info_t * pcdp_summary) { int arg_i = 2; unsigned long rra_begin; /* byte pointer to the rra * area in the rrd file. this * pointer never changes value */ rrd_value_t *pdp_new; /* prepare the incoming data to be added * to the existing entry */ rrd_value_t *pdp_temp; /* prepare the pdp values to be added * to the cdp values */ long *tmpl_idx; /* index representing the settings * transported by the tmplt index */ unsigned long tmpl_cnt = 2; /* time and data */ rrd_t rrd; time_t current_time = 0; unsigned long current_time_usec = 0; /* microseconds part of current time */ char **updvals; int schedule_smooth = 0; /* number of elapsed PDP steps since last update */ unsigned long *rra_step_cnt = NULL; int version; /* rrd version */ rrd_file_t *rrd_file; char *arg_copy; /* for processing the argv */ unsigned long *skip_update; /* RRAs to advance but not write */ int process_ret; /* need at least 1 arguments: data. */ if (argc < 1) { rrd_set_error("Not enough arguments"); goto err_out; } rrd_init(&rrd); rrd_file = rrd_open(filename, &rrd, RRD_READWRITE | RRD_LOCK); if (rrd_file == NULL) { goto err_free; } /* We are now at the beginning of the rra's */ rra_begin = rrd_file->header_len; version = atoi(rrd.stat_head->version); initialize_time(¤t_time, ¤t_time_usec, version); if (allocate_data_structures(&rrd, &updvals, &pdp_temp, tmplt, &tmpl_idx, &tmpl_cnt, &rra_step_cnt, &skip_update, &pdp_new) == -1) { goto err_close; } /* loop through the arguments. */ for (arg_i = 0; arg_i < argc; arg_i++) { if ((arg_copy = strdup(argv[arg_i])) == NULL) { rrd_set_error("failed duplication argv entry"); break; } process_ret = process_arg(arg_copy, &rrd, rrd_file, rra_begin, ¤t_time, ¤t_time_usec, pdp_temp, pdp_new, rra_step_cnt, updvals, tmpl_idx, tmpl_cnt, &pcdp_summary, version, skip_update, &schedule_smooth); if ( ( process_ret == -1 ) || ( ! (extra_flags & RRD_SKIP_PAST_UPDATES) && process_ret == -2 ) ) { if (rrd_test_error()) { /* Should have error string always here */ char *save_error; /* Prepend file name to error message */ if ((save_error = strdup(rrd_get_error())) != NULL) { rrd_set_error("%s: %s", filename, save_error); free(save_error); } else rrd_set_error("error message was lost (out of memory)"); } free(arg_copy); break; } if ( process_ret == -2 ){ rrd_clear_error(); } free(arg_copy); } free(rra_step_cnt); /* if we got here and if there is an error and if the file has not been * written to, then close things up and return. */ if (rrd_test_error()) { goto err_free_structures; } #ifdef HAVE_LIBRADOS if (rrd_file->rados) write_changes_to_disk(&rrd, rrd_file, version); #endif #ifndef HAVE_MMAP if (write_changes_to_disk(&rrd, rrd_file, version) == -1) { goto err_free_structures; } #endif /* calling the smoothing code here guarantees at most one smoothing * operation per rrd_update call. Unfortunately, it is possible with bulk * updates, or a long-delayed update for smoothing to occur off-schedule. * This really isn't critical except during the burn-in cycles. */ if (schedule_smooth) { smooth_all_rras(&rrd, rrd_file, rra_begin); } /* rrd_dontneed(rrd_file,&rrd); */ rrd_free(&rrd); rrd_close(rrd_file); free(pdp_new); free(tmpl_idx); free(pdp_temp); free(skip_update); free(updvals); return 0; err_free_structures: free(pdp_new); free(tmpl_idx); free(pdp_temp); free(skip_update); free(updvals); err_close: rrd_close(rrd_file); err_free: rrd_free(&rrd); err_out: return -1; } /* * Allocate some important arrays used, and initialize the template. * * When it returns, either all of the structures are allocated * or none of them are. * * Returns 0 on success, -1 on error. */ static int allocate_data_structures( rrd_t *rrd, char ***updvals, rrd_value_t **pdp_temp, const char *tmplt, long **tmpl_idx, unsigned long *tmpl_cnt, unsigned long **rra_step_cnt, unsigned long **skip_update, rrd_value_t **pdp_new) { unsigned i, ii; if ((*updvals = (char **) malloc(sizeof(char *) * (rrd->stat_head->ds_cnt + 1))) == NULL) { rrd_set_error("allocating updvals pointer array."); return -1; } if ((*pdp_temp = (rrd_value_t *) malloc(sizeof(rrd_value_t) * rrd->stat_head->ds_cnt)) == NULL) { rrd_set_error("allocating pdp_temp."); goto err_free_updvals; } if ((*skip_update = (unsigned long *) malloc(sizeof(unsigned long) * rrd->stat_head->rra_cnt)) == NULL) { rrd_set_error("allocating skip_update."); goto err_free_pdp_temp; } if ((*tmpl_idx = (long *) malloc(sizeof(unsigned long) * (rrd->stat_head->ds_cnt + 1))) == NULL) { rrd_set_error("allocating tmpl_idx."); goto err_free_skip_update; } if ((*rra_step_cnt = (unsigned long *) malloc(sizeof(unsigned long) * (rrd->stat_head-> rra_cnt))) == NULL) { rrd_set_error("allocating rra_step_cnt."); goto err_free_tmpl_idx; } /* initialize tmplt redirector */ /* default config example (assume DS 1 is a CDEF DS) tmpl_idx[0] -> 0; (time) tmpl_idx[1] -> 1; (DS 0) tmpl_idx[2] -> 3; (DS 2) tmpl_idx[3] -> 4; (DS 3) */ (*tmpl_idx)[0] = 0; /* time */ for (i = 1, ii = 1; i <= rrd->stat_head->ds_cnt; i++) { if (dst_conv(rrd->ds_def[i - 1].dst) != DST_CDEF) (*tmpl_idx)[ii++] = i; } *tmpl_cnt = ii; if (tmplt != NULL) { if (parse_template(rrd, tmplt, tmpl_cnt, *tmpl_idx) == -1) { goto err_free_rra_step_cnt; } } if ((*pdp_new = (rrd_value_t *) malloc(sizeof(rrd_value_t) * rrd->stat_head->ds_cnt)) == NULL) { rrd_set_error("allocating pdp_new."); goto err_free_rra_step_cnt; } return 0; err_free_rra_step_cnt: free(*rra_step_cnt); err_free_tmpl_idx: free(*tmpl_idx); err_free_skip_update: free(*skip_update); err_free_pdp_temp: free(*pdp_temp); err_free_updvals: free(*updvals); return -1; } /* * Parses tmplt and puts an ordered list of DS's into tmpl_idx. * * Returns 0 on success. */ static int parse_template( rrd_t *rrd, const char *tmplt, unsigned long *tmpl_cnt, long *tmpl_idx) { char *dsname, *tmplt_copy; unsigned int tmpl_len, i; int ret = 0; *tmpl_cnt = 1; /* the first entry is the time */ /* we should work on a writeable copy here */ if ((tmplt_copy = strdup(tmplt)) == NULL) { rrd_set_error("error copying tmplt '%s'", tmplt); ret = -1; goto out; } dsname = tmplt_copy; tmpl_len = strlen(tmplt_copy); for (i = 0; i <= tmpl_len; i++) { if (tmplt_copy[i] == ':' || tmplt_copy[i] == '\0') { tmplt_copy[i] = '\0'; if (*tmpl_cnt > rrd->stat_head->ds_cnt) { rrd_set_error("tmplt contains more DS definitions than RRD"); ret = -1; goto out_free_tmpl_copy; } if ((tmpl_idx[(*tmpl_cnt)++] = ds_match(rrd, dsname) + 1) == 0) { rrd_set_error("unknown DS name '%s'", dsname); ret = -1; goto out_free_tmpl_copy; } /* go to the next entry on the tmplt_copy */ if (i < tmpl_len) dsname = &tmplt_copy[i + 1]; } } out_free_tmpl_copy: free(tmplt_copy); out: return ret; } /* * Parse an update string, updates the primary data points (PDPs) * and consolidated data points (CDPs), and writes changes to the RRAs. * * Returns 0 on success, -1 on error, -2 on time stamp error. */ static int process_arg( char *step_start, rrd_t *rrd, rrd_file_t *rrd_file, unsigned long rra_begin, time_t *current_time, unsigned long *current_time_usec, rrd_value_t *pdp_temp, rrd_value_t *pdp_new, unsigned long *rra_step_cnt, char **updvals, long *tmpl_idx, unsigned long tmpl_cnt, rrd_info_t ** pcdp_summary, int version, unsigned long *skip_update, int *schedule_smooth) { rrd_value_t *seasonal_coef = NULL, *last_seasonal_coef = NULL; /* a vector of future Holt-Winters seasonal coefs */ unsigned long elapsed_pdp_st; double interval, pre_int, post_int; /* interval between this and * the last run */ unsigned long proc_pdp_cnt; int ds_ret; if ((ds_ret = parse_ds(rrd, updvals, tmpl_idx, step_start, tmpl_cnt, current_time, current_time_usec, version)) != 0) { return ds_ret; } interval = (double) (*current_time - rrd->live_head->last_up) + (double) ((long) *current_time_usec - (long) rrd->live_head->last_up_usec) / 1e6f; /* process the data sources and update the pdp_prep * area accordingly */ if (update_pdp_prep(rrd, updvals, pdp_new, interval) == -1) { return -1; } elapsed_pdp_st = calculate_elapsed_steps(rrd, *current_time, *current_time_usec, interval, &pre_int, &post_int, &proc_pdp_cnt); /* has a pdp_st moment occurred since the last run ? */ if (elapsed_pdp_st == 0) { /* no we have not passed a pdp_st moment. therefore update is simple */ simple_update(rrd, interval, pdp_new); } else { /* JKammler - bugfix start */ /* additional processing may be needed if elapsed time closes more than one PDP */ if(elapsed_pdp_st > 1) { unsigned long sec_open_pdp; /* Seconds belonging to the most left open PDP */ /* process the most left PDP separately if it was open */ if((sec_open_pdp = ((unsigned long) pre_int) % rrd->stat_head->pdp_step) > 0) { /* allocate temp storage for assigning values to most left open pdp */ rrd_value_t *open_pdp_new; if ((open_pdp_new = (rrd_value_t *) malloc(sizeof(rrd_value_t) * rrd->stat_head->ds_cnt)) == NULL) { rrd_set_error("allocating open_pdp_new."); return -1; } /* assign ds-values proportionally to most left open PDP and all others */ for (unsigned int ds_idx = 0; ds_idx < rrd->stat_head->ds_cnt; ds_idx++) { if(!isnan(pdp_new[ds_idx]) && interval > 0) { open_pdp_new[ds_idx] = pdp_new[ds_idx] * (double) sec_open_pdp / interval; pdp_new[ds_idx] -= open_pdp_new[ds_idx]; } else { open_pdp_new[ds_idx] = DNAN; } #ifdef DEBUG fprintf(stderr, "Split values ds[%lu]\t" "left_open_pdp %5.2f\t" "right_other_pdp %5.2f\n", ds_idx, open_pdp_new[ds_idx], pdp_new[ds_idx]); #endif } if (process_all_pdp_st(rrd, (double) sec_open_pdp, /* interval */ (double) sec_open_pdp, /* pre_int */ 0.0, /* post_int */ 1L, /* elapsed_pdp_st */ open_pdp_new, pdp_temp) == -1) { return -1; } if (update_all_cdp_prep(rrd, rra_step_cnt, rra_begin, rrd_file, 1L, /* elapsed_pdp_st */ proc_pdp_cnt, &last_seasonal_coef, &seasonal_coef, pdp_temp, skip_update, schedule_smooth) == -1) { goto err_free_coefficients; } if (update_aberrant_cdps(rrd, rrd_file, rra_begin, 1L, /* elapsed_pdp_st */ pdp_temp, &seasonal_coef) == -1) { goto err_free_coefficients; } if (write_to_rras(rrd, rrd_file, rra_step_cnt, rra_begin, (rrd->live_head->last_up + sec_open_pdp), /* left PDP close time */ skip_update, pcdp_summary) == -1) { goto err_free_coefficients; } /* modify times for processing remaining PDP's */ interval -= sec_open_pdp; pre_int -= sec_open_pdp; elapsed_pdp_st--; proc_pdp_cnt++; free(open_pdp_new); } } /* JKammler - bugfix end */ /* an pdp_st has occurred. */ if (process_all_pdp_st(rrd, interval, pre_int, post_int, elapsed_pdp_st, pdp_new, pdp_temp) == -1) { return -1; } if (update_all_cdp_prep(rrd, rra_step_cnt, rra_begin, rrd_file, elapsed_pdp_st, proc_pdp_cnt, &last_seasonal_coef, &seasonal_coef, pdp_temp, skip_update, schedule_smooth) == -1) { goto err_free_coefficients; } if (update_aberrant_cdps(rrd, rrd_file, rra_begin, elapsed_pdp_st, pdp_temp, &seasonal_coef) == -1) { goto err_free_coefficients; } if (write_to_rras(rrd, rrd_file, rra_step_cnt, rra_begin, *current_time, skip_update, pcdp_summary) == -1) { goto err_free_coefficients; } } /* endif a pdp_st has occurred */ rrd->live_head->last_up = *current_time; rrd->live_head->last_up_usec = *current_time_usec; if (version < 3) { *rrd->legacy_last_up = rrd->live_head->last_up; } free(seasonal_coef); free(last_seasonal_coef); return 0; err_free_coefficients: free(seasonal_coef); free(last_seasonal_coef); return -1; } /* * Parse a DS string (time + colon-separated values), storing the * results in current_time, current_time_usec, and updvals. * * Returns 0 on success, -1 on error. */ static int parse_ds( rrd_t *rrd, char **updvals, long *tmpl_idx, char *input, unsigned long tmpl_cnt, time_t *current_time, unsigned long *current_time_usec, int version) { char *p; unsigned long i; char timesyntax; updvals[0] = input; /* initialize all ds input to unknown except the first one which has always got to be set */ for (i = 1; i <= rrd->stat_head->ds_cnt; i++) updvals[i] = "U"; /* separate all ds elements; first must be examined separately due to alternate time syntax */ if ((p = strchr(input, '@')) != NULL) { timesyntax = '@'; } else if ((p = strchr(input, ':')) != NULL) { timesyntax = ':'; } else { rrd_set_error("expected timestamp not found in data source from %s", input); return -1; } *p = '\0'; i = 1; updvals[tmpl_idx[i++]] = p + 1; while (*(++p)) { if (*p == ':') { *p = '\0'; if (i < tmpl_cnt) { updvals[tmpl_idx[i++]] = p + 1; } else { rrd_set_error("found extra data on update argument: %s",p+1); return -1; } } } if (i != tmpl_cnt) { rrd_set_error("expected %lu data source readings (got %lu) from %s", tmpl_cnt - 1, i - 1, input); return -1; } return get_time_from_reading(rrd, timesyntax, updvals, current_time, current_time_usec, version); } /* * Parse the time in a DS string, store it in current_time and * current_time_usec and verify that it's later than the last * update for this DS. * * Returns 0 on success, -1 on error. */ static int get_time_from_reading( rrd_t *rrd, char timesyntax, char **updvals, time_t *current_time, unsigned long *current_time_usec, int version) { double tmp; char *parsetime_error = NULL; rrd_time_value_t ds_tv; struct timeval tmp_time; /* used for time conversion */ /* get the time from the reading ... handle N */ if (timesyntax == '@') { /* at-style */ if ((parsetime_error = rrd_parsetime(updvals[0], &ds_tv))) { rrd_set_error("ds time: %s: %s", updvals[0], parsetime_error); return -1; } if (ds_tv.type == RELATIVE_TO_END_TIME || ds_tv.type == RELATIVE_TO_START_TIME) { rrd_set_error("specifying time relative to the 'start' " "or 'end' makes no sense here: %s", updvals[0]); return -1; } *current_time = mktime(&ds_tv.tm) +ds_tv.offset; *current_time_usec = 0; /* FIXME: how to handle usecs here ? */ } else if (strcmp(updvals[0], "N") == 0) { gettimeofday(&tmp_time, 0); normalize_time(&tmp_time); *current_time = tmp_time.tv_sec; *current_time_usec = tmp_time.tv_usec; } else { if ( rrd_strtodbl( updvals[0], NULL, &tmp, "error while parsing time in get_time_from_reading") != 2) { return -1; }; if (tmp < 0.0){ gettimeofday(&tmp_time, 0); tmp = (double)tmp_time.tv_sec + (double)tmp_time.tv_usec * 1e-6f + tmp; } *current_time = floor(tmp); *current_time_usec = (long) ((tmp - (double) *current_time) * 1e6f); } /* dont do any correction for old version RRDs */ if (version < 3) *current_time_usec = 0; if (*current_time < rrd->live_head->last_up || (*current_time == rrd->live_head->last_up && (long) *current_time_usec <= (long) rrd->live_head->last_up_usec)) { rrd_set_error("illegal attempt to update using time %ld when " "last update time is %ld (minimum one second step)", *current_time, rrd->live_head->last_up); return -2; } return 0; } /* * Update pdp_new by interpreting the updvals according to the DS type * (COUNTER, GAUGE, etc.). * * Returns 0 on success, -1 on error. */ static int update_pdp_prep( rrd_t *rrd, char **updvals, rrd_value_t *pdp_new, double interval) { unsigned long ds_idx; int ii; double rate, newval, oldval; enum dst_en dst_idx; for (ds_idx = 0; ds_idx < rrd->stat_head->ds_cnt; ds_idx++) { dst_idx = dst_conv(rrd->ds_def[ds_idx].dst); /* make sure we do not build diffs with old last_ds values */ if (rrd->ds_def[ds_idx].par[DS_mrhb_cnt].u_cnt < interval) { strncpy(rrd->pdp_prep[ds_idx].last_ds, "U", LAST_DS_LEN - 1); rrd->pdp_prep[ds_idx].last_ds[LAST_DS_LEN - 1] = '\0'; } /* NOTE: DST_CDEF should never enter this if block, because * updvals[ds_idx+1][0] is initialized to 'U'; unless the caller * accidently specified a value for the DST_CDEF. To handle this case, * an extra check is required. */ if ((updvals[ds_idx + 1][0] != 'U') && (dst_idx != DST_CDEF) && rrd->ds_def[ds_idx].par[DS_mrhb_cnt].u_cnt >= interval) { rate = DNAN; /* pdp_new contains rate * time ... eg the bytes transferred during * the interval. Doing it this way saves a lot of math operations */ switch (dst_idx) { case DST_COUNTER: case DST_DERIVE: /* Check if this is a valid integer. `U' is already handled in * another branch. */ for (ii = 0; updvals[ds_idx + 1][ii] != 0; ii++) { if ((ii == 0) && (dst_idx == DST_DERIVE) && (updvals[ds_idx + 1][ii] == '-')) continue; if ((updvals[ds_idx + 1][ii] < '0') || (updvals[ds_idx + 1][ii] > '9')) { rrd_set_error("not a simple %s integer: '%s'", (dst_idx == DST_DERIVE) ? "signed" : "unsigned", updvals[ds_idx + 1]); return -1; } } /* for (ii = 0; updvals[ds_idx + 1][ii] != 0; ii++) */ if (rrd->pdp_prep[ds_idx].last_ds[0] != 'U') { pdp_new[ds_idx] = rrd_diff(updvals[ds_idx + 1], rrd->pdp_prep[ds_idx].last_ds); if (dst_idx == DST_COUNTER) { /* simple overflow catcher. This will fail * terribly for non 32 or 64 bit counters * ... are there any others in SNMP land? */ if (pdp_new[ds_idx] < (double) 0.0) pdp_new[ds_idx] += (double) 4294967295.0; /* 2^32-1 */ if (pdp_new[ds_idx] < (double) 0.0) pdp_new[ds_idx] += (double) 18446744069414584320.0; /* 2^64-2^32 */ } rate = pdp_new[ds_idx] / interval; } else { pdp_new[ds_idx] = DNAN; } break; case DST_ABSOLUTE: if( rrd_strtodbl(updvals[ds_idx + 1], NULL, &newval, "Function update_pdp_prep, case DST_ABSOLUTE" ) != 2 ) { return -1; } pdp_new[ds_idx] = newval; rate = pdp_new[ds_idx] / interval; break; case DST_GAUGE: if( rrd_strtodbl( updvals[ds_idx + 1], NULL, &newval, "Function update_pdp_prep, case DST_GAUGE") != 2 ) { return -1; } pdp_new[ds_idx] = newval * interval; rate = newval; break; case DST_DCOUNTER: case DST_DDERIVE: if (rrd->pdp_prep[ds_idx].last_ds[0] != 'U') { const char *e_msg; if (dst_idx == DST_DCOUNTER) { e_msg = "Function update_pdp_prep, case DST_DCOUNTER"; } else { e_msg = "Function update_pdp_prep, case DST_DDERIVE"; } if (rrd_strtodbl(updvals[ds_idx + 1], NULL, &newval, e_msg) != 2) { return -1; } if (rrd_strtodbl(rrd->pdp_prep[ds_idx].last_ds, NULL, &oldval, e_msg) != 2) { return -1; } if (dst_idx == DST_DCOUNTER) { /* * DST_DCOUNTER is always signed, so it can count either up, * or down, but not both at the same time. Changing direction * considered a "reset". */ if ((newval > 0 && oldval > newval) || (newval < 0 && newval > oldval)) { /* Counter reset detected */ pdp_new[ds_idx] = DNAN; break; } } pdp_new[ds_idx] = newval - oldval; rate = pdp_new[ds_idx] / interval; } else { pdp_new[ds_idx] = DNAN; } break; default: rrd_set_error("rrd contains unknown DS type : '%s'", rrd->ds_def[ds_idx].dst); return -1; } /* break out of this for loop if the error string is set */ if (rrd_test_error()) { return -1; } /* make sure pdp_temp is neither too large or too small * if any of these occur it becomes unknown ... * sorry folks ... */ if (!isnan(rate) && ((!isnan(rrd->ds_def[ds_idx].par[DS_max_val].u_val) && rate > rrd->ds_def[ds_idx].par[DS_max_val].u_val) || (!isnan(rrd->ds_def[ds_idx].par[DS_min_val].u_val) && rate < rrd->ds_def[ds_idx].par[DS_min_val].u_val))) { pdp_new[ds_idx] = DNAN; } } else { /* no news is news all the same */ pdp_new[ds_idx] = DNAN; } /* make a copy of the command line argument for the next run */ #ifdef DEBUG fprintf(stderr, "prep ds[%lu]\t" "last_arg '%s'\t" "this_arg '%s'\t" "pdp_new %10.2f\n", ds_idx, rrd->pdp_prep[ds_idx].last_ds, updvals[ds_idx + 1], pdp_new[ds_idx]); #endif strncpy(rrd->pdp_prep[ds_idx].last_ds, updvals[ds_idx + 1], LAST_DS_LEN - 1); rrd->pdp_prep[ds_idx].last_ds[LAST_DS_LEN - 1] = '\0'; } return 0; } /* * How many PDP steps have elapsed since the last update? Returns the answer, * and stores the time between the last update and the last PDP in pre_time, * and the time between the last PDP and the current time in post_int. */ static int calculate_elapsed_steps( rrd_t *rrd, unsigned long current_time, unsigned long current_time_usec, double interval, double *pre_int, double *post_int, unsigned long *proc_pdp_cnt) { unsigned long proc_pdp_st; /* which pdp_st was the last to be processed */ unsigned long occu_pdp_st; /* when was the pdp_st before the last update * time */ unsigned long proc_pdp_age; /* how old was the data in the pdp prep area * when it was last updated */ unsigned long occu_pdp_age; /* how long ago was the last pdp_step time */ /* when was the current pdp started */ proc_pdp_age = rrd->live_head->last_up % rrd->stat_head->pdp_step; proc_pdp_st = rrd->live_head->last_up - proc_pdp_age; /* when did the last pdp_st occur */ occu_pdp_age = current_time % rrd->stat_head->pdp_step; occu_pdp_st = current_time - occu_pdp_age; if (occu_pdp_st > proc_pdp_st) { /* OK we passed the pdp_st moment */ *pre_int = (long) occu_pdp_st - rrd->live_head->last_up; /* how much of the input data * occurred before the latest * pdp_st moment*/ *pre_int -= ((double) rrd->live_head->last_up_usec) / 1e6f; /* adjust usecs */ *post_int = occu_pdp_age; /* how much after it */ *post_int += ((double) current_time_usec) / 1e6f; /* adjust usecs */ } else { *pre_int = interval; *post_int = 0; } *proc_pdp_cnt = proc_pdp_st / rrd->stat_head->pdp_step; #ifdef DEBUG printf("proc_pdp_age %lu\t" "proc_pdp_st %lu\t" "occu_pfp_age %lu\t" "occu_pdp_st %lu\t" "int %lf\t" "pre_int %lf\t" "post_int %lf\n", proc_pdp_age, proc_pdp_st, occu_pdp_age, occu_pdp_st, interval, *pre_int, *post_int); #endif /* compute the number of elapsed pdp_st moments */ return (occu_pdp_st - proc_pdp_st) / rrd->stat_head->pdp_step; } /* * Increment the PDP values by the values in pdp_new, or else initialize them. */ static void simple_update( rrd_t *rrd, double interval, rrd_value_t *pdp_new) { int i; for (i = 0; i < (signed) rrd->stat_head->ds_cnt; i++) { if (isnan(pdp_new[i])) { /* this is not really accurate if we use subsecond data arrival time should have thought of it when going subsecond resolution ... sorry next format change we will have it! */ rrd->pdp_prep[i].scratch[PDP_unkn_sec_cnt].u_cnt += floor(interval); } else { if (isnan(rrd->pdp_prep[i].scratch[PDP_val].u_val)) { rrd->pdp_prep[i].scratch[PDP_val].u_val = pdp_new[i]; } else { rrd->pdp_prep[i].scratch[PDP_val].u_val += pdp_new[i]; } } #ifdef DEBUG fprintf(stderr, "NO PDP ds[%i]\t" "value %10.2f\t" "unkn_sec %5lu\n", i, rrd->pdp_prep[i].scratch[PDP_val].u_val, rrd->pdp_prep[i].scratch[PDP_unkn_sec_cnt].u_cnt); #endif } } /* * Call process_pdp_st for each DS. * * Returns 0 on success, -1 on error. */ static int process_all_pdp_st( rrd_t *rrd, double interval, double pre_int, double post_int, unsigned long elapsed_pdp_st, rrd_value_t *pdp_new, rrd_value_t *pdp_temp) { unsigned long ds_idx; /* in pdp_prep[].scratch[PDP_val].u_val we have collected rate*seconds which occurred up to the last run. pdp_new[] contains rate*seconds from the latest run. pdp_temp[] will contain the rate for cdp */ for (ds_idx = 0; ds_idx < rrd->stat_head->ds_cnt; ds_idx++) { if (process_pdp_st(rrd, ds_idx, interval, pre_int, post_int, elapsed_pdp_st * rrd->stat_head->pdp_step, pdp_new, pdp_temp) == -1) { return -1; } #ifdef DEBUG fprintf(stderr, "PDP UPD ds[%lu]\t" "elapsed_pdp_st %lu\t" "pdp_temp %10.2f\t" "new_prep %10.2f\t" "new_unkn_sec %5lu\n", ds_idx, elapsed_pdp_st, pdp_temp[ds_idx], rrd->pdp_prep[ds_idx].scratch[PDP_val].u_val, rrd->pdp_prep[ds_idx].scratch[PDP_unkn_sec_cnt].u_cnt); #endif } return 0; } /* * Process an update that occurs after one of the PDP moments. * Increments the PDP value, sets NAN if time greater than the * heartbeats have elapsed, processes CDEFs. * * Returns 0 on success, -1 on error. */ static int process_pdp_st( rrd_t *rrd, unsigned long ds_idx, double interval, double pre_int, double post_int, long diff_pdp_st, /* number of seconds in full steps passed since last update */ rrd_value_t *pdp_new, rrd_value_t *pdp_temp) { int i; /* update pdp_prep to the current pdp_st. */ double pre_unknown = 0.0; unival *scratch = rrd->pdp_prep[ds_idx].scratch; unsigned long mrhb = rrd->ds_def[ds_idx].par[DS_mrhb_cnt].u_cnt; rpnstack_t rpnstack; /* used for COMPUTE DS */ rpnstack_init(&rpnstack); if (isnan(pdp_new[ds_idx])) { /* a final bit of unknown to be added before calculation we use a temporary variable for this so that we don't have to turn integer lines before using the value */ pre_unknown = pre_int; } else { if (isnan(scratch[PDP_val].u_val)) { scratch[PDP_val].u_val = 0; } scratch[PDP_val].u_val += pdp_new[ds_idx] / interval * pre_int; } /* if too much of the pdp_prep is unknown we dump it */ /* if the interval is larger than mrhb we get NAN */ if ((interval > mrhb) || (rrd->stat_head->pdp_step / 2.0 < (signed) scratch[PDP_unkn_sec_cnt].u_cnt)) { pdp_temp[ds_idx] = DNAN; } else { pdp_temp[ds_idx] = scratch[PDP_val].u_val / ((double) (diff_pdp_st - scratch[PDP_unkn_sec_cnt].u_cnt) - pre_unknown); } /* process CDEF data sources; remember each CDEF DS can * only reference other DS with a lower index number */ if (dst_conv(rrd->ds_def[ds_idx].dst) == DST_CDEF) { rpnp_t *rpnp; rpnp = rpn_expand((rpn_cdefds_t *) &(rrd->ds_def[ds_idx].par[DS_cdef])); if(rpnp == NULL) { rpnstack_free(&rpnstack); return -1; } /* substitute data values for OP_VARIABLE nodes */ for (i = 0; rpnp[i].op != OP_END; i++) { if (rpnp[i].op == OP_VARIABLE) { rpnp[i].op = OP_NUMBER; rpnp[i].val = pdp_temp[rpnp[i].ptr]; } /* just in case */ rpnp[i].extra = NULL; rpnp[i].free_extra = NULL; } /* run the rpn calculator */ if (rpn_calc(rpnp, &rpnstack, 0, pdp_temp, ds_idx,rrd->stat_head->pdp_step) == -1) { rpnp_freeextra(rpnp); free(rpnp); rpnstack_free(&rpnstack); return -1; } rpnp_freeextra(rpnp); free(rpnp); } /* make pdp_prep ready for the next run */ if (isnan(pdp_new[ds_idx])) { /* this is not really accurate if we use subsecond data arrival time should have thought of it when going subsecond resolution ... sorry next format change we will have it! */ scratch[PDP_unkn_sec_cnt].u_cnt = floor(post_int); scratch[PDP_val].u_val = DNAN; } else { scratch[PDP_unkn_sec_cnt].u_cnt = 0; scratch[PDP_val].u_val = pdp_new[ds_idx] / interval * post_int; } rpnstack_free(&rpnstack); return 0; } /* * Iterate over all the RRAs for a given DS and: * 1. Decide whether to schedule a smooth later * 2. Decide whether to skip updating SEASONAL and DEVSEASONAL * 3. Update the CDP * * Returns 0 on success, -1 on error */ static int update_all_cdp_prep( rrd_t *rrd, unsigned long *rra_step_cnt, unsigned long rra_begin, rrd_file_t *rrd_file, unsigned long elapsed_pdp_st, unsigned long proc_pdp_cnt, rrd_value_t **last_seasonal_coef, rrd_value_t **seasonal_coef, rrd_value_t *pdp_temp, unsigned long *skip_update, int *schedule_smooth) { unsigned long rra_idx; /* index into the CDP scratch array */ enum cf_en current_cf; unsigned long rra_start; /* number of rows to be updated in an RRA for a data value. */ unsigned long start_pdp_offset; rra_start = rra_begin; for (rra_idx = 0; rra_idx < rrd->stat_head->rra_cnt; rra_idx++) { current_cf = rrd_cf_conv(rrd->rra_def[rra_idx].cf_nam); start_pdp_offset = rrd->rra_def[rra_idx].pdp_cnt - proc_pdp_cnt % rrd->rra_def[rra_idx].pdp_cnt; skip_update[rra_idx] = 0; if (start_pdp_offset <= elapsed_pdp_st) { rra_step_cnt[rra_idx] = min((elapsed_pdp_st - start_pdp_offset) / rrd->rra_def[rra_idx].pdp_cnt + 1, rrd->rra_def[rra_idx].row_cnt); } else { rra_step_cnt[rra_idx] = 0; } if (current_cf == CF_SEASONAL || current_cf == CF_DEVSEASONAL) { /* If this is a bulk update, we need to skip ahead in the seasonal arrays * so that they will be correct for the next observed value; note that for * the bulk update itself, no update will occur to DEVSEASONAL or SEASONAL; * furthermore, HWPREDICT and DEVPREDICT will be set to DNAN. */ if (rra_step_cnt[rra_idx] > 1) { skip_update[rra_idx] = 1; lookup_seasonal(rrd, rra_idx, rra_start, rrd_file, elapsed_pdp_st, last_seasonal_coef); lookup_seasonal(rrd, rra_idx, rra_start, rrd_file, elapsed_pdp_st + 1, seasonal_coef); } /* periodically run a smoother for seasonal effects */ if (do_schedule_smooth(rrd, rra_idx, elapsed_pdp_st)) { #ifdef DEBUG fprintf(stderr, "schedule_smooth: cur_row %lu, elapsed_pdp_st %lu, smooth idx %lu\n", rrd->rra_ptr[rra_idx].cur_row, elapsed_pdp_st, rrd->rra_def[rra_idx].par[RRA_seasonal_smooth_idx]. u_cnt); #endif *schedule_smooth = 1; } } if (rrd_test_error()) return -1; if (update_cdp_prep (rrd, elapsed_pdp_st, start_pdp_offset, rra_step_cnt, rra_idx, pdp_temp, *last_seasonal_coef, *seasonal_coef, current_cf) == -1) { return -1; } rra_start += rrd->rra_def[rra_idx].row_cnt * rrd->stat_head->ds_cnt * sizeof(rrd_value_t); } return 0; } /* * Are we due for a smooth? Also increments our position in the burn-in cycle. */ static int do_schedule_smooth( rrd_t *rrd, unsigned long rra_idx, unsigned long elapsed_pdp_st) { unsigned long cdp_idx = rra_idx * (rrd->stat_head->ds_cnt); unsigned long cur_row = rrd->rra_ptr[rra_idx].cur_row; unsigned long row_cnt = rrd->rra_def[rra_idx].row_cnt; unsigned long seasonal_smooth_idx = rrd->rra_def[rra_idx].par[RRA_seasonal_smooth_idx].u_cnt; unsigned long *init_seasonal = &(rrd->cdp_prep[cdp_idx].scratch[CDP_init_seasonal].u_cnt); /* Need to use first cdp parameter buffer to track burnin (burnin requires * a specific smoothing schedule). The CDP_init_seasonal parameter is * really an RRA level, not a data source within RRA level parameter, but * the rra_def is read only for rrd_update (not flushed to disk). */ if (*init_seasonal > BURNIN_CYCLES) { /* someone has no doubt invented a trick to deal with this wrap around, * but at least this code is clear. */ if (seasonal_smooth_idx > cur_row) { /* here elapsed_pdp_st = rra_step_cnt[rra_idx] because of 1-1 mapping * between PDP and CDP */ return (cur_row + elapsed_pdp_st >= seasonal_smooth_idx); } /* can't rely on negative numbers because we are working with * unsigned values */ return (cur_row + elapsed_pdp_st >= row_cnt && cur_row + elapsed_pdp_st >= row_cnt + seasonal_smooth_idx); } /* mark off one of the burn-in cycles */ return (cur_row + elapsed_pdp_st >= row_cnt && ++(*init_seasonal)); } /* * For a given RRA, iterate over the data sources and call the appropriate * consolidation function. * * Returns 0 on success, -1 on error. */ static int update_cdp_prep( rrd_t *rrd, unsigned long elapsed_pdp_st, unsigned long start_pdp_offset, unsigned long *rra_step_cnt, int rra_idx, rrd_value_t *pdp_temp, rrd_value_t *last_seasonal_coef, rrd_value_t *seasonal_coef, int current_cf) { unsigned long ds_idx, cdp_idx; /* update CDP_PREP areas */ /* loop over data soures within each RRA */ for (ds_idx = 0; ds_idx < rrd->stat_head->ds_cnt; ds_idx++) { cdp_idx = rra_idx * rrd->stat_head->ds_cnt + ds_idx; if (rrd->rra_def[rra_idx].pdp_cnt > 1) { update_cdp(rrd->cdp_prep[cdp_idx].scratch, current_cf, pdp_temp[ds_idx], rra_step_cnt[rra_idx], elapsed_pdp_st, start_pdp_offset, rrd->rra_def[rra_idx].pdp_cnt, rrd->rra_def[rra_idx].par[RRA_cdp_xff_val].u_val, rra_idx, ds_idx); } else { /* Nothing to consolidate if there's one PDP per CDP. However, if * we've missed some PDPs, let's update null counters etc. */ if (elapsed_pdp_st > 2) { reset_cdp(rrd, elapsed_pdp_st, pdp_temp, last_seasonal_coef, seasonal_coef, rra_idx, ds_idx, cdp_idx, (enum cf_en)current_cf); } } if (rrd_test_error()) return -1; } /* endif data sources loop */ return 0; } /* * Given the new reading (pdp_temp_val), update or initialize the CDP value, * primary value, secondary value, and # of unknowns. */ void update_cdp( unival *scratch, int current_cf, rrd_value_t pdp_temp_val, unsigned long rra_step_cnt, unsigned long elapsed_pdp_st, unsigned long start_pdp_offset, unsigned long pdp_cnt, rrd_value_t xff, int i, int ii) { /* shorthand variables */ rrd_value_t *cdp_val = &scratch[CDP_val].u_val; rrd_value_t *cdp_primary_val = &scratch[CDP_primary_val].u_val; rrd_value_t *cdp_secondary_val = &scratch[CDP_secondary_val].u_val; unsigned long *cdp_unkn_pdp_cnt = &scratch[CDP_unkn_pdp_cnt].u_cnt; if (rra_step_cnt) { /* If we are in this block, as least 1 CDP value will be written to * disk, this is the CDP_primary_val entry. If more than 1 value needs * to be written, then the "fill in" value is the CDP_secondary_val * entry. */ if (isnan(pdp_temp_val)) { *cdp_unkn_pdp_cnt += start_pdp_offset; *cdp_secondary_val = DNAN; } else { /* CDP_secondary value is the RRA "fill in" value for intermediary * CDP data entries. No matter the CF, the value is the same because * the average, max, min, and last of a list of identical values is * the same, namely, the value itself. */ *cdp_secondary_val = pdp_temp_val; } if (*cdp_unkn_pdp_cnt > pdp_cnt * xff) { *cdp_primary_val = DNAN; } else { initialize_cdp_val(scratch, current_cf, pdp_temp_val, start_pdp_offset, pdp_cnt); } *cdp_val = initialize_carry_over(pdp_temp_val,current_cf, elapsed_pdp_st, start_pdp_offset, pdp_cnt); /* endif meets xff value requirement for a valid value */ /* initialize carry over CDP_unkn_pdp_cnt, this must after CDP_primary_val * is set because CDP_unkn_pdp_cnt is required to compute that value. */ if (isnan(pdp_temp_val)) *cdp_unkn_pdp_cnt = (elapsed_pdp_st - start_pdp_offset) % pdp_cnt; else *cdp_unkn_pdp_cnt = 0; } else { /* rra_step_cnt[i] == 0 */ #ifdef DEBUG if (isnan(*cdp_val)) { fprintf(stderr, "schedule CDP_val update, RRA %d DS %d, DNAN\n", i, ii); } else { fprintf(stderr, "schedule CDP_val update, RRA %d DS %d, %10.2f\n", i, ii, *cdp_val); } #endif if (isnan(pdp_temp_val)) { *cdp_unkn_pdp_cnt += elapsed_pdp_st; } else { *cdp_val = calculate_cdp_val(*cdp_val, pdp_temp_val, elapsed_pdp_st, current_cf, i, ii); } } } /* * Set the CDP_primary_val and CDP_val to the appropriate initial value based * on the type of consolidation function. */ static void initialize_cdp_val( unival *scratch, int current_cf, rrd_value_t pdp_temp_val, unsigned long start_pdp_offset, unsigned long pdp_cnt) { rrd_value_t cum_val, cur_val; switch (current_cf) { case CF_AVERAGE: cum_val = IFDNAN(scratch[CDP_val].u_val, 0.0); cur_val = IFDNAN(pdp_temp_val, 0.0); scratch[CDP_primary_val].u_val = (cum_val + cur_val * start_pdp_offset) / (pdp_cnt - scratch[CDP_unkn_pdp_cnt].u_cnt); break; case CF_MAXIMUM: cum_val = IFDNAN(scratch[CDP_val].u_val, -DINF); cur_val = IFDNAN(pdp_temp_val, -DINF); #if 0 #ifdef DEBUG if (isnan(scratch[CDP_val].u_val) && isnan(pdp_temp)) { fprintf(stderr, "RRA %lu, DS %lu, both CDP_val and pdp_temp are DNAN!", i, ii); break; } #endif #endif if (cur_val > cum_val) scratch[CDP_primary_val].u_val = cur_val; else scratch[CDP_primary_val].u_val = cum_val; break; case CF_MINIMUM: cum_val = IFDNAN(scratch[CDP_val].u_val, DINF); cur_val = IFDNAN(pdp_temp_val, DINF); #if 0 #ifdef DEBUG if (isnan(scratch[CDP_val].u_val) && isnan(pdp_temp)) { fprintf(stderr, "RRA %lu, DS %lu, both CDP_val and pdp_temp are DNAN!", i, ii); break; } #endif #endif if (cur_val < cum_val) scratch[CDP_primary_val].u_val = cur_val; else scratch[CDP_primary_val].u_val = cum_val; break; case CF_LAST: default: scratch[CDP_primary_val].u_val = pdp_temp_val; break; } } /* * Update the consolidation function for Holt-Winters functions as * well as other functions that don't actually consolidate multiple * PDPs. */ static void reset_cdp( rrd_t *rrd, unsigned long elapsed_pdp_st, rrd_value_t *pdp_temp, rrd_value_t *last_seasonal_coef, rrd_value_t *seasonal_coef, int rra_idx, int ds_idx, int cdp_idx, enum cf_en current_cf) { unival *scratch = rrd->cdp_prep[cdp_idx].scratch; switch (current_cf) { case CF_AVERAGE: default: scratch[CDP_primary_val].u_val = pdp_temp[ds_idx]; scratch[CDP_secondary_val].u_val = pdp_temp[ds_idx]; break; case CF_SEASONAL: case CF_DEVSEASONAL: /* need to update cached seasonal values, so they are consistent * with the bulk update */ /* WARNING: code relies on the fact that CDP_hw_last_seasonal and * CDP_last_deviation are the same. */ scratch[CDP_hw_last_seasonal].u_val = last_seasonal_coef[ds_idx]; scratch[CDP_hw_seasonal].u_val = seasonal_coef[ds_idx]; break; case CF_HWPREDICT: case CF_MHWPREDICT: /* need to update the null_count and last_null_count. * even do this for non-DNAN pdp_temp because the * algorithm is not learning from batch updates. */ scratch[CDP_null_count].u_cnt += elapsed_pdp_st; scratch[CDP_last_null_count].u_cnt += elapsed_pdp_st - 1; /* fall through */ case CF_DEVPREDICT: scratch[CDP_primary_val].u_val = DNAN; scratch[CDP_secondary_val].u_val = DNAN; break; case CF_FAILURES: /* do not count missed bulk values as failures */ scratch[CDP_primary_val].u_val = 0; scratch[CDP_secondary_val].u_val = 0; /* need to reset violations buffer. * could do this more carefully, but for now, just * assume a bulk update wipes away all violations. */ erase_violations(rrd, cdp_idx, rra_idx); break; } } static rrd_value_t initialize_carry_over( rrd_value_t pdp_temp_val, int current_cf, unsigned long elapsed_pdp_st, unsigned long start_pdp_offset, unsigned long pdp_cnt) { unsigned long pdp_into_cdp_cnt = ((elapsed_pdp_st - start_pdp_offset) % pdp_cnt); if ( pdp_into_cdp_cnt == 0 || isnan(pdp_temp_val)){ switch (current_cf) { case CF_MAXIMUM: return -DINF; case CF_MINIMUM: return DINF; case CF_AVERAGE: return 0; default: return DNAN; } } else { switch (current_cf) { case CF_AVERAGE: return pdp_temp_val * pdp_into_cdp_cnt ; default: return pdp_temp_val; } } } /* * Update or initialize a CDP value based on the consolidation * function. * * Returns the new value. */ static rrd_value_t calculate_cdp_val( rrd_value_t cdp_val, rrd_value_t pdp_temp_val, unsigned long elapsed_pdp_st, int current_cf, #ifdef DEBUG int i, int ii #else int UNUSED(i), int UNUSED(ii) #endif ) { if (isnan(cdp_val)) { if (current_cf == CF_AVERAGE) { pdp_temp_val *= elapsed_pdp_st; } #ifdef DEBUG fprintf(stderr, "Initialize CDP_val for RRA %d DS %d: %10.2f\n", i, ii, pdp_temp_val); #endif return pdp_temp_val; } if (current_cf == CF_AVERAGE) return cdp_val + pdp_temp_val * elapsed_pdp_st; if (current_cf == CF_MINIMUM) return (pdp_temp_val < cdp_val) ? pdp_temp_val : cdp_val; if (current_cf == CF_MAXIMUM) return (pdp_temp_val > cdp_val) ? pdp_temp_val : cdp_val; return pdp_temp_val; } /* * For each RRA, update the seasonal values and then call update_aberrant_CF * for each data source. * * Return 0 on success, -1 on error. */ static int update_aberrant_cdps( rrd_t *rrd, rrd_file_t *rrd_file, unsigned long rra_begin, unsigned long elapsed_pdp_st, rrd_value_t *pdp_temp, rrd_value_t **seasonal_coef) { unsigned long rra_idx, ds_idx, j; /* number of PDP steps since the last update that * are assigned to the first CDP to be generated * since the last update. */ unsigned short scratch_idx; unsigned long rra_start; enum cf_en current_cf; /* this loop is only entered if elapsed_pdp_st < 3 */ for (j = elapsed_pdp_st, scratch_idx = CDP_primary_val; j > 0 && j < 3; j--, scratch_idx = CDP_secondary_val) { rra_start = rra_begin; for (rra_idx = 0; rra_idx < rrd->stat_head->rra_cnt; rra_idx++) { if (rrd->rra_def[rra_idx].pdp_cnt == 1) { current_cf = rrd_cf_conv(rrd->rra_def[rra_idx].cf_nam); if (current_cf == CF_SEASONAL || current_cf == CF_DEVSEASONAL) { if (scratch_idx == CDP_primary_val) { lookup_seasonal(rrd, rra_idx, rra_start, rrd_file, elapsed_pdp_st + 1, seasonal_coef); } else { lookup_seasonal(rrd, rra_idx, rra_start, rrd_file, elapsed_pdp_st + 2, seasonal_coef); } } if (rrd_test_error()) return -1; /* loop over data sources within each RRA */ for (ds_idx = 0; ds_idx < rrd->stat_head->ds_cnt; ds_idx++) { update_aberrant_CF(rrd, pdp_temp[ds_idx], current_cf, rra_idx * (rrd->stat_head->ds_cnt) + ds_idx, rra_idx, ds_idx, scratch_idx, *seasonal_coef); } } rra_start += rrd->rra_def[rra_idx].row_cnt * rrd->stat_head->ds_cnt * sizeof(rrd_value_t); } } return 0; } /* * Move sequentially through the file, writing one RRA at a time. Note this * architecture divorces the computation of CDP with flushing updated RRA * entries to disk. * * Return 0 on success, -1 on error. */ static int write_to_rras( rrd_t *rrd, rrd_file_t *rrd_file, unsigned long *rra_step_cnt, unsigned long rra_begin, time_t current_time, unsigned long *skip_update, rrd_info_t ** pcdp_summary) { unsigned long rra_idx; unsigned long rra_start; time_t rra_time = 0; /* time of update for a RRA */ unsigned long ds_cnt = rrd->stat_head->ds_cnt; /* Ready to write to disk */ rra_start = rra_begin; for (rra_idx = 0; rra_idx < rrd->stat_head->rra_cnt; rra_idx++) { rra_def_t *rra_def = &rrd->rra_def[rra_idx]; rra_ptr_t *rra_ptr = &rrd->rra_ptr[rra_idx]; /* for cdp_prep */ unsigned short scratch_idx; unsigned long step_subtract; for (scratch_idx = CDP_primary_val, step_subtract = 1; rra_step_cnt[rra_idx] > 0; rra_step_cnt[rra_idx]--, scratch_idx = CDP_secondary_val, step_subtract = 2) { size_t rra_pos_new; #ifdef DEBUG fprintf(stderr, " -- RRA Preseek %ld\n", rrd_file->pos); #endif /* increment, with wrap-around */ if (++rra_ptr->cur_row >= rra_def->row_cnt) rra_ptr->cur_row = 0; /* we know what our position should be */ rra_pos_new = rra_start + ds_cnt * rra_ptr->cur_row * sizeof(rrd_value_t); /* re-seek if the position is wrong or we wrapped around */ if ((size_t)rra_pos_new != rrd_file->pos) { if (rrd_seek(rrd_file, rra_pos_new, SEEK_SET) != 0) { rrd_set_error("seek error in rrd"); return -1; } } #ifdef DEBUG fprintf(stderr, " -- RRA Postseek %ld\n", rrd_file->pos); #endif if (skip_update[rra_idx]) continue; if (*pcdp_summary != NULL) { unsigned long step_time = rra_def->pdp_cnt * rrd->stat_head->pdp_step; rra_time = (current_time - current_time % step_time) - ((rra_step_cnt[rra_idx] - step_subtract) * step_time); } if (write_RRA_row (rrd_file, rrd, rra_idx, scratch_idx, pcdp_summary, rra_time) == -1) return -1; rrd_notify_row(rrd_file, rra_idx, rra_pos_new, rra_time); } rra_start += rra_def->row_cnt * ds_cnt * sizeof(rrd_value_t); } /* RRA LOOP */ return 0; } /* * Write out one row of values (one value per DS) to the archive. * * Returns 0 on success, -1 on error. */ static int write_RRA_row( rrd_file_t *rrd_file, rrd_t *rrd, unsigned long rra_idx, unsigned short CDP_scratch_idx, rrd_info_t ** pcdp_summary, time_t rra_time) { unsigned long ds_idx, cdp_idx; rrd_infoval_t iv; for (ds_idx = 0; ds_idx < rrd->stat_head->ds_cnt; ds_idx++) { /* compute the cdp index */ cdp_idx = rra_idx * (rrd->stat_head->ds_cnt) + ds_idx; #ifdef DEBUG fprintf(stderr, " -- RRA WRITE VALUE %e, at %ld CF:%s\n", rrd->cdp_prep[cdp_idx].scratch[CDP_scratch_idx].u_val, rrd_file->pos, rrd->rra_def[rra_idx].cf_nam); #endif if (*pcdp_summary != NULL) { iv.u_val = rrd->cdp_prep[cdp_idx].scratch[CDP_scratch_idx].u_val; /* append info to the return hash */ *pcdp_summary = rrd_info_push(*pcdp_summary, sprintf_alloc ("[%lli]RRA[%s][%lu]DS[%s]", (long long)rra_time, rrd->rra_def[rra_idx].cf_nam, rrd->rra_def[rra_idx].pdp_cnt, rrd->ds_def[ds_idx].ds_nam), RD_I_VAL, iv); } errno = 0; if (rrd_write(rrd_file, &(rrd->cdp_prep[cdp_idx].scratch[CDP_scratch_idx]. u_val), sizeof(rrd_value_t)) != sizeof(rrd_value_t)) { rrd_set_error("writing rrd: %s", rrd_strerror(errno)); return -1; } } return 0; } /* * Call apply_smoother for all DEVSEASONAL and SEASONAL RRAs. * * Returns 0 on success, -1 otherwise */ static int smooth_all_rras( rrd_t *rrd, rrd_file_t *rrd_file, unsigned long rra_begin) { unsigned long rra_start = rra_begin; unsigned long rra_idx; for (rra_idx = 0; rra_idx < rrd->stat_head->rra_cnt; ++rra_idx) { if (rrd_cf_conv(rrd->rra_def[rra_idx].cf_nam) == CF_DEVSEASONAL || rrd_cf_conv(rrd->rra_def[rra_idx].cf_nam) == CF_SEASONAL) { #ifdef DEBUG fprintf(stderr, "Running smoother for rra %lu\n", rra_idx); #endif apply_smoother(rrd, rra_idx, rra_start, rrd_file); if (rrd_test_error()) return -1; } rra_start += rrd->rra_def[rra_idx].row_cnt * rrd->stat_head->ds_cnt * sizeof(rrd_value_t); } return 0; } #if !defined(HAVE_MMAP) || defined(HAVE_LIBRADOS) /* * Flush changes to disk (unless we're using mmap) * * Returns 0 on success, -1 otherwise */ static int write_changes_to_disk( rrd_t *rrd, rrd_file_t *rrd_file, int version) { /* we just need to write back the live header portion now */ if (rrd_seek(rrd_file, (sizeof(stat_head_t) + sizeof(ds_def_t) * rrd->stat_head->ds_cnt + sizeof(rra_def_t) * rrd->stat_head->rra_cnt), SEEK_SET) != 0) { rrd_set_error("seek rrd for live header writeback"); return -1; } if (version >= 3) { if (rrd_write(rrd_file, rrd->live_head, sizeof(live_head_t) * 1) != sizeof(live_head_t) * 1) { rrd_set_error("rrd_write live_head to rrd"); return -1; } } else { if (rrd_write(rrd_file, rrd->legacy_last_up, sizeof(time_t) * 1) != sizeof(time_t) * 1) { rrd_set_error("rrd_write live_head to rrd"); return -1; } } if (rrd_write(rrd_file, rrd->pdp_prep, sizeof(pdp_prep_t) * rrd->stat_head->ds_cnt) != (ssize_t) (sizeof(pdp_prep_t) * rrd->stat_head->ds_cnt)) { rrd_set_error("rrd_write pdp_prep to rrd"); return -1; } if (rrd_write(rrd_file, rrd->cdp_prep, sizeof(cdp_prep_t) * rrd->stat_head->rra_cnt * rrd->stat_head->ds_cnt) != (ssize_t) (sizeof(cdp_prep_t) * rrd->stat_head->rra_cnt * rrd->stat_head->ds_cnt)) { rrd_set_error("rrd_write cdp_prep to rrd"); return -1; } if (rrd_write(rrd_file, rrd->rra_ptr, sizeof(rra_ptr_t) * rrd->stat_head->rra_cnt) != (ssize_t) (sizeof(rra_ptr_t) * rrd->stat_head->rra_cnt)) { rrd_set_error("rrd_write rra_ptr to rrd"); return -1; } return 0; } #endif /* * vim: set sw=4 sts=4 ts=8 et fdm=marker : */