From 11eb67eec9b4d990ae4df680cf7db77dad1b8630 Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Thu, 25 Jun 2020 19:59:19 -0700 Subject: [PATCH] Some memory allocation improvements - All the memory-allocation macros now auto-check for failure and exit with a failure message that incudes the caller's file and lineno info. This includes strdup(). - Added the `--max-alloc=SIZE` option to be able to override the memory allocator's sanity-check limit. It defaults to 1G (as before). Fixes bugzilla bug 12769. --- NEWS.md | 13 +++++- access.c | 10 ++--- acls.c | 16 ++----- authenticate.c | 7 ++- checksum.c | 2 - clientserver.c | 6 +-- compat.c | 3 +- exclude.c | 21 +++------ fileio.c | 7 +-- flist.c | 30 +++++-------- generator.c | 2 - getgroups.c | 3 +- hashtable.c | 8 ++-- hlink.c | 9 ++-- ifuncs.h | 13 ++++-- io.c | 12 ++--- loadparm.c | 9 ++-- main.c | 25 +++-------- match.c | 2 - options.c | 118 ++++++++++++++++++++++++++----------------------- params.c | 21 --------- rsync.1.md | 41 ++++++++++++++--- rsync.h | 16 ++++--- sender.c | 10 ++--- socket.c | 11 +---- t_stub.c | 1 + token.c | 34 +++++--------- uidlist.c | 10 ----- util.c | 38 ++++++---------- util2.c | 43 +++++++++++++----- xattrs.c | 16 +------ 31 files changed, 239 insertions(+), 318 deletions(-) diff --git a/NEWS.md b/NEWS.md index ae283d574..622dc09b7 100644 --- a/NEWS.md +++ b/NEWS.md @@ -25,11 +25,22 @@ Protocol: 31 (unchanged) - Do not allow a negotiated checksum or compression choice of "none" unless the user authorized it via an environment variable or command-line option. - - Improved the man page a bit more. + - Added the `--max-alloc=SIZE` option to be able to override the memory + allocator's sanity-check limit. It defaults to 1G (as before) but the error + message when exceeding it specifically mentions the new option so that you + can differentiate an out-of-memory error from a failure of this limit. It + also allows you to specify the value via the RSYNC_MAX_ALLOC environment + variable. + + - The memory allocation functions now automatically check for a failure and + die when out of memory. This eliminated some caller-side check-and-die + code and added some missing sanity-checking of allocations. - Preparing for an upcoming xxHash release that provides new XXH3 & XXH128 hashing routines (disabled until their code is finalized). + - Improved the man page a bit more. + ------------------------------------------------------------------------------ diff --git a/access.c b/access.c index 5b6629016..d7bf01cca 100644 --- a/access.c +++ b/access.c @@ -19,6 +19,7 @@ */ #include "rsync.h" +#include "ifuncs.h" static int allow_forward_dns; @@ -52,10 +53,8 @@ static int match_hostname(const char **host_ptr, const char *addr, const char *t if (strcmp(addr, inet_ntoa(*(struct in_addr*)(hp->h_addr_list[i]))) == 0) { /* If reverse lookups are off, we'll use the conf-specified * hostname in preference to UNDETERMINED. */ - if (host == undetermined_hostname) { - if (!(*host_ptr = strdup(tok))) - *host_ptr = undetermined_hostname; - } + if (host == undetermined_hostname) + *host_ptr = strdup(tok); return 1; } } @@ -241,9 +240,6 @@ static int access_match(const char *list, const char *addr, const char **host_pt char *tok; char *list2 = strdup(list); - if (!list2) - out_of_memory("access_match"); - strlower(list2); for (tok = strtok(list2, " ,\t"); tok; tok = strtok(NULL, " ,\t")) { diff --git a/acls.c b/acls.c index 0b5dec6ad..4303c2a74 100644 --- a/acls.c +++ b/acls.c @@ -168,8 +168,6 @@ static rsync_acl *create_racl(void) { rsync_acl *racl = new(rsync_acl); - if (!racl) - out_of_memory("create_racl"); *racl = empty_rsync_acl; return racl; @@ -335,8 +333,7 @@ static BOOL unpack_smb_acl(SMB_ACL_T sacl, rsync_acl *racl) qsort(temp_ida_list.items, temp_ida_list.count, sizeof (id_access), id_access_sorter); } #endif - if (!(racl->names.idas = new_array(id_access, temp_ida_list.count))) - out_of_memory("unpack_smb_acl"); + racl->names.idas = new_array(id_access, temp_ida_list.count); memcpy(racl->names.idas, temp_ida_list.items, temp_ida_list.count * sizeof (id_access)); } else racl->names.idas = NULL; @@ -505,9 +502,7 @@ static int get_rsync_acl(const char *fname, rsync_acl *racl, if (cnt) { char *bp = buf + 4*4; - id_access *ida; - if (!(ida = racl->names.idas = new_array(id_access, cnt))) - out_of_memory("get_rsync_acl"); + id_access *ida = racl->names.idas = new_array(id_access, cnt); racl->names.count = cnt; for ( ; cnt--; ida++, bp += 4+4) { ida->id = IVAL(bp, 0); @@ -703,12 +698,7 @@ static uchar recv_ida_entries(int f, ida_entries *ent) uchar computed_mask_bits = 0; int i, count = read_varint(f); - if (count) { - if (!(ent->idas = new_array(id_access, count))) - out_of_memory("recv_ida_entries"); - } else - ent->idas = NULL; - + ent->idas = count ? new_array(id_access, count) : NULL; ent->count = count; for (i = 0; i < count; i++) { diff --git a/authenticate.c b/authenticate.c index 169331e54..3ef83ef2a 100644 --- a/authenticate.c +++ b/authenticate.c @@ -20,6 +20,7 @@ #include "rsync.h" #include "itypes.h" +#include "ifuncs.h" extern int read_only; extern char *password_file; @@ -250,8 +251,7 @@ char *auth_server(int f_in, int f_out, int module, const char *host, } *pass++ = '\0'; - if (!(users = strdup(users))) - out_of_memory("auth_server"); + users = strdup(users); for (tok = strtok(users, " ,\t"); tok; tok = strtok(NULL, " ,\t")) { char *opts; @@ -287,8 +287,7 @@ char *auth_server(int f_in, int f_out, int module, const char *host, else { gid_t *gid_array = gid_list.items; auth_uid_groups_cnt = gid_list.count; - if ((auth_uid_groups = new_array(char *, auth_uid_groups_cnt)) == NULL) - out_of_memory("auth_server"); + auth_uid_groups = new_array(char *, auth_uid_groups_cnt); for (j = 0; j < auth_uid_groups_cnt; j++) auth_uid_groups[j] = gid_to_group(gid_array[j]); } diff --git a/checksum.c b/checksum.c index 824159b02..b3989aa20 100644 --- a/checksum.c +++ b/checksum.c @@ -271,8 +271,6 @@ void get_checksum2(char *buf, int32 len, char *sum) free(buf1); buf1 = new_array(char, len+4); len1 = len; - if (!buf1) - out_of_memory("get_checksum2"); } memcpy(buf1, buf, len); diff --git a/clientserver.c b/clientserver.c index b790974c3..57bb1b9a0 100644 --- a/clientserver.c +++ b/clientserver.c @@ -235,8 +235,7 @@ int start_inband_exchange(int f_in, int f_out, const char *user, int argc, char else modlen = p - *argv; - if (!(modname = new_array(char, modlen+1+1))) /* room for '/' & '\0' */ - out_of_memory("start_inband_exchange"); + modname = new_array(char, modlen+1+1); /* room for '/' & '\0' */ strlcpy(modname, *argv, modlen + 1); modname[modlen] = '/'; modname[modlen+1] = '\0'; @@ -1233,8 +1232,7 @@ int start_daemon(int f_in, int f_out) io_printf(f_out, "@ERROR: invalid early_input length\n"); return -1; } - if (!(early_input = new_array(char, early_input_len))) - out_of_memory("exchange_protocols"); + early_input = new_array(char, early_input_len); read_buf(f_in, early_input, early_input_len); if (!read_line_old(f_in, line, sizeof line, 0)) diff --git a/compat.c b/compat.c index 11965f71e..ba14a8c52 100644 --- a/compat.c +++ b/compat.c @@ -243,8 +243,7 @@ static void init_nno_saw(struct name_num_obj *nno, int val) } if (!nno->saw) { - if (!(nno->saw = new_array0(uchar, nno->saw_len))) - out_of_memory("init_nno_saw"); + nno->saw = new_array0(uchar, nno->saw_len); /* We'll take this opportunity to make sure that the main_name values are set right. */ for (cnt = 1, nni = nno->list; nni->name; nni++, cnt++) { diff --git a/exclude.c b/exclude.c index df56e802f..10b56e763 100644 --- a/exclude.c +++ b/exclude.c @@ -22,6 +22,7 @@ #include "rsync.h" #include "default-cvsignore.h" +#include "ifuncs.h" extern int am_server; extern int am_sender; @@ -200,8 +201,7 @@ static void add_rule(filter_rule_list *listp, const char *pat, unsigned int pat_ } else suf_len = 0; - if (!(rule->pattern = new_array(char, pre_len + pat_len + suf_len + 1))) - out_of_memory("add_rule"); + rule->pattern = new_array(char, pre_len + pat_len + suf_len + 1); if (pre_len) { memcpy(rule->pattern, dirbuf + module_dirlen, pre_len); for (cp = rule->pattern; cp < rule->pattern + pre_len; cp++) { @@ -262,19 +262,14 @@ static void add_rule(filter_rule_list *listp, const char *pat, unsigned int pat_ } } - if (!(lp = new_array0(filter_rule_list, 1))) - out_of_memory("add_rule"); + lp = new_array0(filter_rule_list, 1); if (asprintf(&lp->debug_type, " [per-dir %s]", cp) < 0) out_of_memory("add_rule"); rule->u.mergelist = lp; if (mergelist_cnt == mergelist_size) { mergelist_size += 5; - mergelist_parents = realloc_array(mergelist_parents, - filter_rule *, - mergelist_size); - if (!mergelist_parents) - out_of_memory("add_rule"); + mergelist_parents = realloc_array(mergelist_parents, filter_rule *, mergelist_size); } if (DEBUG_GTE(FILTER, 2)) { rprintf(FINFO, "[%s] activating mergelist #%d%s\n", @@ -498,8 +493,6 @@ void *push_local_filters(const char *dir, unsigned int dirlen) push = (struct local_filter_state *)new_array(char, sizeof (struct local_filter_state) + (mergelist_cnt-1) * sizeof (filter_rule_list)); - if (!push) - out_of_memory("push_local_filters"); push->mergelist_cnt = mergelist_cnt; for (i = 0; i < mergelist_cnt; i++) { @@ -822,8 +815,7 @@ static filter_rule *parse_rule_tok(const char **rulestr_ptr, if (!*s) return NULL; - if (!(rule = new0(filter_rule))) - out_of_memory("parse_rule_tok"); + rule = new0(filter_rule); /* Inherit from the template. Don't inherit FILTRULES_SIDES; we check * that later. */ @@ -1125,8 +1117,7 @@ void parse_filter_str(filter_rule_list *listp, const char *rulestr, const char *name; filter_rule *excl_self; - if (!(excl_self = new0(filter_rule))) - out_of_memory("parse_filter_str"); + excl_self = new0(filter_rule); /* Find the beginning of the basename and add an exclude for it. */ for (name = pat + pat_len; name > pat && name[-1] != '/'; name--) {} add_rule(listp, name, (pat + pat_len) - name, excl_self, 0); diff --git a/fileio.c b/fileio.c index 32dc62da9..f80af19e4 100644 --- a/fileio.c +++ b/fileio.c @@ -157,8 +157,6 @@ int write_file(int f, int use_seek, OFF_T offset, const char *buf, int len) wf_writeBufSize = WRITE_SIZE * 8; wf_writeBufCnt = 0; wf_writeBuf = new_array(char, wf_writeBufSize); - if (!wf_writeBuf) - out_of_memory("write_file"); } r1 = (int)MIN((size_t)len, wf_writeBufSize - wf_writeBufCnt); if (r1) { @@ -217,8 +215,7 @@ struct map_struct *map_file(int fd, OFF_T len, int32 read_size, int32 blk_size) { struct map_struct *map; - if (!(map = new0(struct map_struct))) - out_of_memory("map_file"); + map = new0(struct map_struct); if (blk_size && (read_size % blk_size)) read_size += blk_size - (read_size % blk_size); @@ -261,8 +258,6 @@ char *map_ptr(struct map_struct *map, OFF_T offset, int32 len) /* make sure we have allocated enough memory for the window */ if (window_size > map->p_size) { map->p = realloc_array(map->p, char, window_size); - if (!map->p) - out_of_memory("map_ptr"); map->p_size = window_size; } diff --git a/flist.c b/flist.c index bbc028bae..6b19776fa 100644 --- a/flist.c +++ b/flist.c @@ -301,8 +301,7 @@ static void flist_expand(struct file_list *flist, int extra) if (flist->malloced < flist->used + extra) flist->malloced = flist->used + extra; - new_ptr = realloc_array(flist->files, struct file_struct *, - flist->malloced); + new_ptr = realloc_array(flist->files, struct file_struct *, flist->malloced); if (DEBUG_GTE(FLIST, 1) && flist->malloced != FLIST_START) { rprintf(FCLIENT, "[%s] expand file_list pointer array to %s bytes, did%s move\n", @@ -1335,10 +1334,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, + linkname_len; if (pool) bp = pool_alloc(pool, alloc_len, "make_file"); - else { - if (!(bp = new_array(char, alloc_len))) - out_of_memory("make_file"); - } + else + bp = new_array(char, alloc_len); memset(bp, 0, extra_len + FILE_STRUCT_LEN); bp += extra_len; @@ -1661,8 +1658,7 @@ static void fsort(struct file_struct **fp, size_t num) if (use_qsort) qsort(fp, num, PTR_SIZE, file_compare); else { - struct file_struct **tmp = new_array(struct file_struct *, - (num+1) / 2); + struct file_struct **tmp = new_array(struct file_struct *, (num+1) / 2); fsort_tmp(fp, num, tmp); free(tmp); } @@ -1895,13 +1891,11 @@ static void send_implied_dirs(int f, struct file_list *flist, char *fname, len = strlen(limit+1); memcpy(&relname_list, F_DIR_RELNAMES_P(lastpath_struct), sizeof relname_list); if (!relname_list) { - if (!(relname_list = new0(item_list))) - out_of_memory("send_implied_dirs"); + relname_list = new0(item_list); memcpy(F_DIR_RELNAMES_P(lastpath_struct), &relname_list, sizeof relname_list); } rnpp = EXPAND_ITEM_LIST(relname_list, relnamecache *, 32); - if (!(*rnpp = (relnamecache*)new_array(char, sizeof (relnamecache) + len))) - out_of_memory("send_implied_dirs"); + *rnpp = (relnamecache*)new_array(char, sizeof (relnamecache) + len); (*rnpp)->name_type = name_type; strlcpy((*rnpp)->fname, limit+1, len + 1); @@ -2059,8 +2053,7 @@ void send_extra_file_list(int f, int at_least) } if (need_unsorted_flist) { - if (!(flist->sorted = new_array(struct file_struct *, flist->used))) - out_of_memory("send_extra_file_list"); + flist->sorted = new_array(struct file_struct *, flist->used); memcpy(flist->sorted, flist->files, flist->used * sizeof (struct file_struct*)); } else @@ -2414,8 +2407,7 @@ struct file_list *send_file_list(int f, int argc, char *argv[]) * recursion mode, the sender marks duplicate dirs so that it can * send them together in a single file-list. */ if (need_unsorted_flist) { - if (!(flist->sorted = new_array(struct file_struct *, flist->used))) - out_of_memory("send_file_list"); + flist->sorted = new_array(struct file_struct *, flist->used); memcpy(flist->sorted, flist->files, flist->used * sizeof (struct file_struct*)); } else @@ -2597,8 +2589,7 @@ struct file_list *recv_file_list(int f, int dir_ndx) * order and for calling flist_find()). We keep the "files" * list unsorted for our exchange of index numbers with the * other side (since their names may not sort the same). */ - if (!(flist->sorted = new_array(struct file_struct *, flist->used))) - out_of_memory("recv_file_list"); + flist->sorted = new_array(struct file_struct *, flist->used); memcpy(flist->sorted, flist->files, flist->used * sizeof (struct file_struct*)); if (inc_recurse && dir_flist->used > dstart) { @@ -2808,8 +2799,7 @@ struct file_list *flist_new(int flags, char *msg) { struct file_list *flist; - if (!(flist = new0(struct file_list))) - out_of_memory(msg); + flist = new0(struct file_list); if (flags & FLIST_TEMP) { if (!(flist->file_pool = pool_create(SMALL_EXTENT, 0, diff --git a/generator.c b/generator.c index 8f9d60359..23a2eaff7 100644 --- a/generator.c +++ b/generator.c @@ -2227,8 +2227,6 @@ void generate_files(int f_out, const char *local_name) if (delete_during == 2) { deldelay_size = BIGPATHBUFLEN * 4; deldelay_buf = new_array(char, deldelay_size); - if (!deldelay_buf) - out_of_memory("delete-delay"); } info_levels[INFO_FLIST] = info_levels[INFO_PROGRESS] = 0; diff --git a/getgroups.c b/getgroups.c index a96e04d4a..1ccbc7812 100644 --- a/getgroups.c +++ b/getgroups.c @@ -20,8 +20,7 @@ #include "rsync.h" -int -main(UNUSED(int argc), UNUSED(char *argv[])) + int main(UNUSED(int argc), UNUSED(char *argv[])) { int n, i; gid_t *list; diff --git a/hashtable.c b/hashtable.c index 17133dd2b..e272f439d 100644 --- a/hashtable.c +++ b/hashtable.c @@ -35,9 +35,8 @@ struct hashtable *hashtable_create(int size, int key64) size *= 2; } - if (!(tbl = new(struct hashtable)) - || !(tbl->nodes = new_array0(char, size * node_size))) - out_of_memory("hashtable_create"); + tbl = new(struct hashtable); + tbl->nodes = new_array0(char, size * node_size); tbl->size = size; tbl->entries = 0; tbl->node_size = node_size; @@ -94,8 +93,7 @@ void *hashtable_find(struct hashtable *tbl, int64 key, void *data_when_new) int size = tbl->size * 2; int i; - if (!(tbl->nodes = new_array0(char, size * tbl->node_size))) - out_of_memory("hashtable_node"); + tbl->nodes = new_array0(char, size * tbl->node_size); tbl->size = size; tbl->entries = 0; diff --git a/hlink.c b/hlink.c index 85f547043..adec89b0d 100644 --- a/hlink.c +++ b/hlink.c @@ -125,8 +125,7 @@ static void match_gnums(int32 *ndx_list, int ndx_count) if (inc_recurse) { node = hashtable_find(prior_hlinks, gnum, data_when_new); if (node->data == data_when_new) { - if (!(node->data = new_array0(char, 5))) - out_of_memory("match_gnums"); + node->data = new_array0(char, 5); assert(gnum >= hlink_flist->ndx_start); file->flags |= FLAG_HLINK_FIRST; prev = -1; @@ -190,8 +189,7 @@ void match_hard_links(struct file_list *flist) int i, ndx_count = 0; int32 *ndx_list; - if (!(ndx_list = new_array(int32, flist->used))) - out_of_memory("match_hard_links"); + ndx_list = new_array(int32, flist->used); for (i = 0; i < flist->used; i++) { if (F_IS_HLINKED(flist->sorted[i])) @@ -541,8 +539,7 @@ void finish_hard_link(struct file_struct *file, const char *fname, int fin_ndx, exit_cleanup(RERR_MESSAGEIO); } free(node->data); - if (!(node->data = strdup(our_name))) - out_of_memory("finish_hard_link"); + node->data = strdup(our_name); } } diff --git a/ifuncs.h b/ifuncs.h index 36ea51ad7..7f9bde09f 100644 --- a/ifuncs.h +++ b/ifuncs.h @@ -19,8 +19,7 @@ static inline void alloc_xbuf(xbuf *xb, size_t sz) { - if (!(xb->buf = new_array(char, sz))) - out_of_memory("alloc_xbuf"); + xb->buf = new_array(char, sz); xb->size = sz; xb->len = xb->pos = 0; } @@ -29,8 +28,6 @@ static inline void realloc_xbuf(xbuf *xb, size_t sz) { char *bf = realloc_array(xb->buf, char, sz); - if (!bf) - out_of_memory("realloc_xbuf"); xb->buf = bf; xb->size = sz; } @@ -104,3 +101,11 @@ free_stat_x(stat_x *sx_p) } #endif } + +static inline char *my_strdup(const char *str, const char *file, int line) +{ + int len = strlen(str)+1; + char *buf = _my_alloc(do_malloc, len, 1, file, line); + memcpy(buf, str, len); + return buf; +} diff --git a/io.c b/io.c index 6c3730c8f..ddd20fa8d 100644 --- a/io.c +++ b/io.c @@ -1239,8 +1239,7 @@ void read_args(int f_in, char *mod_name, char *buf, size_t bufsiz, int rl_nulls, rl_flags |= (protect_args && ic_recv != (iconv_t)-1 ? RL_CONVERT : 0); #endif - if (!(argv = new_array(char *, maxargs))) - out_of_memory("read_args"); + argv = new_array(char *, maxargs); if (mod_name && !protect_args) argv[argc++] = "rsyncd"; @@ -1253,8 +1252,7 @@ void read_args(int f_in, char *mod_name, char *buf, size_t bufsiz, int rl_nulls, if (argc == maxargs-1) { maxargs += MAX_ARGS; - if (!(argv = realloc_array(argv, char *, maxargs))) - out_of_memory("read_args"); + argv = realloc_array(argv, char *, maxargs); } if (dot_pos) { @@ -1262,8 +1260,7 @@ void read_args(int f_in, char *mod_name, char *buf, size_t bufsiz, int rl_nulls, int len = strlen(buf); if (request_len) request_p[0][request_len++] = ' '; - if (!(*request_p = realloc_array(*request_p, char, request_len + len + 1))) - out_of_memory("read_args"); + *request_p = realloc_array(*request_p, char, request_len + len + 1); memcpy(*request_p + request_len, buf, len + 1); request_len += len; } @@ -1272,8 +1269,7 @@ void read_args(int f_in, char *mod_name, char *buf, size_t bufsiz, int rl_nulls, else glob_expand(buf, &argv, &argc, &maxargs); } else { - if (!(p = strdup(buf))) - out_of_memory("read_args"); + p = strdup(buf); argv[argc++] = p; if (*p == '.' && p[1] == '\0') dot_pos = argc; diff --git a/loadparm.c b/loadparm.c index dfc0b0778..819888c5d 100644 --- a/loadparm.c +++ b/loadparm.c @@ -42,6 +42,7 @@ #include "rsync.h" #include "itypes.h" +#include "ifuncs.h" #include "default-dont-compress.h" extern item_list dparam_list; @@ -471,8 +472,7 @@ static char *expand_vars(char *str) return str; bufsize = strlen(str) + 2048; - if ((buf = new_array(char, bufsize+1)) == NULL) /* +1 for trailing '\0' */ - out_of_memory("expand_vars"); + buf = new_array(char, bufsize+1); /* +1 for trailing '\0' */ for (t = buf, f = str; bufsize && *f; ) { if (*f == '%' && *++f != '%') { @@ -601,10 +601,7 @@ FN_LOCAL_BOOL(lp_write_only, write_only) * the start, so any lost memory is inconsequential. */ static inline void string_set(char **s, const char *v) { - if (!v) - *s = NULL; - else if (!(*s = strdup(v))) - out_of_memory("string_set"); + *s = v ? strdup(v) : NULL; } /* Copy local_vars into a new section. No need to strdup since we don't free. */ diff --git a/main.c b/main.c index b41a39435..c832d575e 100644 --- a/main.c +++ b/main.c @@ -22,6 +22,7 @@ #include "rsync.h" #include "inums.h" +#include "ifuncs.h" #include "io.h" #if defined CONFIG_LOCALE && defined HAVE_LOCALE_H #include @@ -512,8 +513,6 @@ static pid_t do_cmd(char *cmd, char *machine, char *user, char **remote_argv, in if (!cmd) cmd = RSYNC_RSH; cmd = need_to_free = strdup(cmd); - if (!cmd) - goto oom; for (t = f = cmd; *f; f++) { if (*f == ' ') @@ -657,10 +656,6 @@ static pid_t do_cmd(char *cmd, char *machine, char *user, char **remote_argv, in free(need_to_free); return pid; - - oom: - out_of_memory("do_cmd"); - return 0; /* not reached */ } /* The receiving side operates in one of two modes: @@ -824,8 +819,6 @@ static void check_alt_basis_dirs(void) if (dry_run > 1 && *bdir != '/') { int len = curr_dir_len + 1 + bd_len + 1; char *new = new_array(char, len); - if (!new) - out_of_memory("check_alt_basis_dirs"); if (slash && strncmp(bdir, "../", 3) == 0) { /* We want to remove only one leading "../" prefix for * the directory we couldn't create in dry-run mode: @@ -1339,19 +1332,12 @@ int client_run(int f_in, int f_out, pid_t pid, int argc, char *argv[]) return MAX(exit_code, exit_code2); } -static int copy_argv(char *argv[]) +static void dup_argv(char *argv[]) { int i; - for (i = 0; argv[i]; i++) { - if (!(argv[i] = strdup(argv[i]))) { - rprintf (FERROR, "out of memory at %s(%d)\n", - __FILE__, __LINE__); - return RERR_MALLOC; - } - } - - return 0; + for (i = 0; argv[i]; i++) + argv[i] = strdup(argv[i]); } @@ -1372,8 +1358,7 @@ static int start_client(int argc, char *argv[]) /* Don't clobber argv[] so that ps(1) can still show the right * command line. */ - if ((ret = copy_argv(argv)) != 0) - return ret; + dup_argv(argv); if (!read_batch) { /* for read_batch, NO source is specified */ char *path = check_for_hostspec(argv[0], &shell_machine, &rsync_port); diff --git a/match.c b/match.c index 4ae7aa469..9d5c92594 100644 --- a/match.c +++ b/match.c @@ -65,8 +65,6 @@ static void build_hash_table(struct sum_struct *s) if (hash_table) free(hash_table); hash_table = new_array(int32, tablesize); - if (!hash_table) - out_of_memory("build_hash_table"); alloc_size = tablesize; } diff --git a/options.c b/options.c index a5038703e..141fa24f7 100644 --- a/options.c +++ b/options.c @@ -21,6 +21,7 @@ #include "rsync.h" #include "itypes.h" +#include "ifuncs.h" #include "latest-year.h" #include @@ -181,6 +182,10 @@ int rsync_port = 0; int alt_dest_type = 0; int basis_dir_cnt = 0; +#define DEFAULT_MAX_ALLOC (1024L * 1024 * 1024) +size_t max_alloc = DEFAULT_MAX_ALLOC; +char *max_alloc_arg; + static int version_opt_cnt = 0; static int remote_option_alloc = 0; int remote_option_cnt = 0; @@ -382,8 +387,7 @@ static char *make_output_option(struct output_struct *words, short *levels, ucha return NULL; len++; - if (!(buf = new_array(char, len))) - out_of_memory("make_output_option"); + buf = new_array(char, len); pos = 0; if (skipped || max < 5) @@ -889,6 +893,7 @@ static struct poptOption long_options[] = { {"ignore-existing", 0, POPT_ARG_NONE, &ignore_existing, 0, 0, 0 }, {"max-size", 0, POPT_ARG_STRING, &max_size_arg, OPT_MAX_SIZE, 0, 0 }, {"min-size", 0, POPT_ARG_STRING, &min_size_arg, OPT_MIN_SIZE, 0, 0 }, + {"max-alloc", 0, POPT_ARG_STRING, &max_alloc_arg, 0, 0, 0 }, {"sparse", 'S', POPT_ARG_VAL, &sparse_files, 1, 0, 0 }, {"no-sparse", 0, POPT_ARG_VAL, &sparse_files, 0, 0, 0 }, {"no-S", 0, POPT_ARG_VAL, &sparse_files, 0, 0, 0 }, @@ -1251,14 +1256,16 @@ static int count_args(const char **argv) return i; } - -static OFF_T parse_size_arg(char **size_arg, char def_suf) +/* If the size_arg is an invalid string or the value is < min_value, an error + * is put into err_buf & the return is -1. Note that this parser does NOT + * support negative numbers, so a min_value < 0 doesn't make any sense. */ +static ssize_t parse_size_arg(char *size_arg, char def_suf, const char *opt_name, ssize_t min_value) { - int reps, mult, make_compatible = 0; - const char *arg; - OFF_T size = 1; + int reps, mult; + const char *arg, *err = "invalid"; + ssize_t size = 1; - for (arg = *size_arg; isDigit(arg); arg++) {} + for (arg = size_arg; isDigit(arg); arg++) {} if (*arg == '.') for (arg++; isDigit(arg); arg++) {} switch (*arg && *arg != '+' && *arg != '-' ? *arg++ : def_suf) { @@ -1274,40 +1281,40 @@ static OFF_T parse_size_arg(char **size_arg, char def_suf) case 'g': case 'G': reps = 3; break; + case 't': case 'T': + reps = 4; + break; + case 'p': case 'P': + reps = 5; + break; default: - return -1; + goto failure; } if (*arg == 'b' || *arg == 'B') - mult = 1000, make_compatible = 1, arg++; + mult = 1000, arg++; else if (!*arg || *arg == '+' || *arg == '-') mult = 1024; else if (strncasecmp(arg, "ib", 2) == 0) mult = 1024, arg += 2; else - return -1; + goto failure; while (reps--) size *= mult; - size *= atof(*size_arg); + size *= atof(size_arg); if ((*arg == '+' || *arg == '-') && arg[1] == '1') - size += atoi(arg), make_compatible = 1, arg += 2; + size += atoi(arg), arg += 2; if (*arg) - return -1; - if (size > 0 && make_compatible && def_suf == 'b') { - /* We convert this manually because we may need %lld precision, - * and that's not a portable sprintf() escape. */ - char buf[128], *s = buf + sizeof buf - 1; - OFF_T num = size; - *s = '\0'; - while (num) { - *--s = (char)(num % 10) + '0'; - num /= 10; - } - if (!(*size_arg = strdup(s))) - out_of_memory("parse_size_arg"); + goto failure; + if (size < min_value) { + err = size < 0 ? "too big" : "too small"; + goto failure; } return size; -} +failure: + snprintf(err_buf, sizeof err_buf, "--%s value is %s: %s\n", opt_name, err, size_arg); + return -1; +} static void create_refuse_error(int which) { @@ -1530,8 +1537,6 @@ int parse_arguments(int *argc_p, const char ***argv_p) if (daemon_filter_list.head) { int rej; char *cp = strdup(arg); - if (!cp) - out_of_memory("parse_arguments"); if (!*cp) rej = 1; else { @@ -1655,8 +1660,6 @@ int parse_arguments(int *argc_p, const char ***argv_p) remote_option_alloc += 16; remote_options = realloc_array(remote_options, const char *, remote_option_alloc); - if (!remote_options) - out_of_memory("parse_arguments"); if (!remote_option_cnt) remote_options[0] = "ARG0"; } @@ -1686,39 +1689,25 @@ int parse_arguments(int *argc_p, const char ***argv_p) break; case OPT_MAX_SIZE: - if ((max_size = parse_size_arg(&max_size_arg, 'b')) < 0) { - snprintf(err_buf, sizeof err_buf, - "--max-size value is invalid: %s\n", - max_size_arg); + if ((max_size = parse_size_arg(max_size_arg, 'b', "max-size", 0)) < 0) return 0; - } + max_size_arg = num_to_byte_string(max_size); break; case OPT_MIN_SIZE: - if ((min_size = parse_size_arg(&min_size_arg, 'b')) < 0) { - snprintf(err_buf, sizeof err_buf, - "--min-size value is invalid: %s\n", - min_size_arg); + if ((min_size = parse_size_arg(min_size_arg, 'b', "min-size", 0)) < 0) return 0; - } + min_size_arg = num_to_byte_string(min_size); break; - case OPT_BWLIMIT: - { - OFF_T limit = parse_size_arg(&bwlimit_arg, 'K'); - if (limit < 0) { - snprintf(err_buf, sizeof err_buf, - "--bwlimit value is invalid: %s\n", bwlimit_arg); - return 0; - } - bwlimit = (limit + 512) / 1024; - if (limit && !bwlimit) { - snprintf(err_buf, sizeof err_buf, - "--bwlimit value is too small: %s\n", bwlimit_arg); - return 0; - } - } + case OPT_BWLIMIT: { + ssize_t size = parse_size_arg(bwlimit_arg, 'K', "bwlimit", 512); + if (size < 0) + return 0; + bwlimit_arg = num_to_byte_string(size); + bwlimit = (size + 512) / 1024; break; + } case OPT_APPEND: if (am_server) @@ -1900,6 +1889,18 @@ int parse_arguments(int *argc_p, const char ***argv_p) exit_cleanup(0); } + if (!max_alloc_arg) { + max_alloc_arg = getenv("RSYNC_MAX_ALLOC"); + if (max_alloc_arg && !*max_alloc_arg) + max_alloc_arg = NULL; + } + if (max_alloc_arg) { + ssize_t size = parse_size_arg(max_alloc_arg, 'B', "max-alloc", 1024*1024); + if (size < 0) + return 0; + max_alloc = size; + } + if (protect_args < 0) { if (am_server) protect_args = 0; @@ -2770,6 +2771,11 @@ void server_options(char **args, int *argc_p) } } + if (max_alloc_arg && max_alloc != DEFAULT_MAX_ALLOC) { + args[ac++] = "--max-alloc"; + args[ac++] = max_alloc_arg; + } + /* --delete-missing-args needs the cooperation of both sides, but * the sender can handle --ignore-missing-args by itself. */ if (missing_args == 2) diff --git a/params.c b/params.c index 910ef233d..da8a8f88a 100644 --- a/params.c +++ b/params.c @@ -212,11 +212,6 @@ static BOOL Section( FILE *InFile, BOOL (*sfunc)(char *) ) { bSize += BUFR_INC; bufr = realloc_array( bufr, char, bSize ); - if( NULL == bufr ) - { - rprintf(FLOG, "%s Memory re-allocation failure.", func); - return( False ); - } } /* Handle a single character. */ @@ -306,11 +301,6 @@ static BOOL Parameter( FILE *InFile, BOOL (*pfunc)(char *, char *), int c ) { bSize += BUFR_INC; bufr = realloc_array( bufr, char, bSize ); - if( NULL == bufr ) - { - rprintf(FLOG, "%s Memory re-allocation failure.", func) ; - return( False ); - } } switch( c ) @@ -382,11 +372,6 @@ static BOOL Parameter( FILE *InFile, BOOL (*pfunc)(char *, char *), int c ) { bSize += BUFR_INC; bufr = realloc_array( bufr, char, bSize ); - if( NULL == bufr ) - { - rprintf(FLOG, "%s Memory re-allocation failure.", func) ; - return( False ); - } } switch( c ) @@ -639,12 +624,6 @@ int pm_process( char *FileName, { /* allocate one, then parse, */ bSize = BUFR_INC; /* then free. */ bufr = new_array( char, bSize ); - if( NULL == bufr ) - { - rprintf(FLOG, "%s memory allocation failure.\n", func); - fclose(InFile); - return( False ); - } result = Parse( InFile, sfunc, pfunc ); free( bufr ); bufr = NULL; diff --git a/rsync.1.md b/rsync.1.md index ebd59dcad..e145efd05 100644 --- a/rsync.1.md +++ b/rsync.1.md @@ -403,6 +403,7 @@ detailed description below for a complete description. --max-delete=NUM don't delete more than NUM files --max-size=SIZE don't transfer any file larger than SIZE --min-size=SIZE don't transfer any file smaller than SIZE +--max-alloc=SIZE change a limit relating to memory alloc --partial keep partially transferred files --partial-dir=DIR put a partially transferred file into DIR --delay-updates put all updated files into place at end @@ -605,7 +606,7 @@ your home directory (remove the '=' for that). helps to avoid overpopulating the protocol data with extra message data. The option does not affect the remote side of a transfer without using - `--remote-option` -- e.g. `-M--msgs2stderr` or `{-M,}--msgs2stderr`. + `--remote-option`, e.g. `-M--msgs2stderr` or `{-M,}--msgs2stderr`. Also keep in mind that connecting to a normal (non-remote-shell) daemon does not have a stderr channel to send messages back to the client side, so @@ -1732,12 +1733,16 @@ your home directory (remove the '=' for that). data that goes into the file-lists, and thus it doesn't affect deletions. It just limits the files that the receiver requests to be transferred. - The suffixes are as follows: "K" (or "KiB") is a kibibyte (1024), "M" (or - "MiB") is a mebibyte (1024\*1024), and "G" (or "GiB") is a gibibyte - (1024\*1024\*1024). If you want the multiplier to be 1000 instead of 1024, - use "KB", "MB", or "GB". (Note: lower-case is also accepted for all - values.) Finally, if the suffix ends in either "+1" or "-1", the value will - be offset by one byte in the indicated direction. + The suffix letters are (in upper/lower-case): `B`, `K`, `G`, `T`, and `P` + for bytes, kilobytes/kibibytes, megabytes/mebibytes, gigabytes/gibibytes, + terabytes/tebibytes, and petabytes/pebibytes. If you use a single-char + suffix or add-on "ib" to it (e.g. "G" or "GiB") then you get units that are + multiples of 1024. If you use a two-letter suffix that ends with a "B" + (e.g. "kb") then you get units that are multiples of 1000. + + Finally, if the value ends with either "+1" or "-1", it will be offset by + one byte in the indicated direction. The largest possible value is + `8192P-1`. Examples: `--max-size=1.5mb-1` is 1499999 bytes, and `--max-size=2g+1` is 2147483649 bytes. @@ -1752,6 +1757,28 @@ your home directory (remove the '=' for that). Note that rsync versions prior to 3.1.0 did not allow `--min-size=0`. +0. `--max-alloc=SIZE` + + By default rsync limits an individual malloc/realloc to about 1GB in size. + For most people this limit works just fine and prevents a code issue + causing rsync to request massive amounts of memory. However, if you have + many millions of files in a transfer, a huge amount of server memory, and + you don't want to split up your transfer into multiple parts, you can + increase the per-allocation limit to something larger and rsync will + consume more memory. + + Keep in mind that this is not a limit on the total size of allocated + memory. It is a sanity-check value for individual allocations. + + See the `--max-size` option for a description of how SIZE can be specified. + The default suffix if none is given is bytes. + + You can set a default value using the environment variable RSYNC_MAX_ALLOC + using the same SIZE values as supported by this option. If the remote + rsync doesn't understand the `--max-alloc` option, you can override the + setting by specifying `--max-alloc=1g` (because rsync will not send the + option to the remote side when the value is the default). + 0. `--block-size=SIZE`, `-B` This forces the block size used in rsync's delta-transfer algorithm to a diff --git a/rsync.h b/rsync.h index a645f2019..ef2668d13 100644 --- a/rsync.h +++ b/rsync.h @@ -1262,12 +1262,18 @@ extern int errno; /* handler for null strings in printf format */ #define NS(s) ((s)?(s):"") +extern char *do_malloc; + /* Convenient wrappers for malloc and realloc. Use them. */ -#define new(type) ((type*)malloc(sizeof (type))) -#define new0(type) ((type*)calloc(1, sizeof (type))) -#define new_array(type, num) ((type*)_new_array((num), sizeof (type), 0)) -#define new_array0(type, num) ((type*)_new_array((num), sizeof (type), 1)) -#define realloc_array(ptr, type, num) ((type*)_realloc_array((ptr), (num), sizeof (type))) +#define new(type) ((type*)_my_alloc(do_malloc, 1, sizeof (type), __FILE__, __LINE__)) +#define new0(type) ((type*)_my_alloc(NULL, 1, sizeof (type), __FILE__, __LINE__)) +#define realloc_buf(ptr, num) _my_alloc((ptr), (num), 1, __FILE__, __LINE__) + +#define new_array(type, num) ((type*)_my_alloc(do_malloc, (num), sizeof (type), __FILE__, __LINE__)) +#define new_array0(type, num) ((type*)_my_alloc(NULL, (num), sizeof (type), __FILE__, __LINE__)) +#define realloc_array(ptr, type, num) ((type*)_my_alloc((ptr), (num), sizeof (type), __FILE__, __LINE__)) + +#define strdup(s) my_strdup(s, __FILE__, __LINE__) /* use magic gcc attributes to catch format errors */ void rprintf(enum logcode , const char *, ...) diff --git a/sender.c b/sender.c index abd79bd2d..94761c262 100644 --- a/sender.c +++ b/sender.c @@ -65,13 +65,10 @@ BOOL extra_flist_sending_enabled; **/ static struct sum_struct *receive_sums(int f) { - struct sum_struct *s; - int32 i; + struct sum_struct *s = new(struct sum_struct); int lull_mod = protocol_version >= 31 ? 0 : allowed_lull * 5; OFF_T offset = 0; - - if (!(s = new(struct sum_struct))) - out_of_memory("receive_sums"); + int32 i; read_sum_head(f, s); @@ -92,8 +89,7 @@ static struct sum_struct *receive_sums(int f) if (s->count == 0) return(s); - if (!(s->sums = new_array(struct sum_buf, s->count))) - out_of_memory("receive_sums"); + s->sums = new_array(struct sum_buf, s->count); for (i = 0; i < s->count; i++) { s->sums[i].sum1 = read_int(f); diff --git a/socket.c b/socket.c index ee6fb6c6f..c2075adf8 100644 --- a/socket.c +++ b/socket.c @@ -26,6 +26,7 @@ #include "rsync.h" #include "itypes.h" +#include "ifuncs.h" #ifdef HAVE_NETINET_IN_SYSTM_H #include #endif @@ -248,8 +249,6 @@ int open_socket_out(char *host, int port, const char *bind_addr, int af_hint) for (res = res0, addr_cnt = 0; res; res = res->ai_next, addr_cnt++) {} errnos = new_array0(int, addr_cnt); - if (!errnos) - out_of_memory("open_socket_out"); s = -1; /* Try to connect to all addresses for this machine until we get @@ -354,8 +353,7 @@ int open_socket_out_wrapped(char *host, int port, const char *bind_addr, int af_ len += hlen; } f = prog; - if (!(prog = new_array(char, len))) - out_of_memory("open_socket_out_wrapped"); + prog = new_array(char, len); for (t = prog; *f; f++) { if (*f == '%') { switch (*++f) { @@ -423,8 +421,6 @@ static int *open_socket_in(int type, int port, const char *bind_addr, socks = new_array(int, maxs + 1); errmsgs = new_array(char *, maxs); - if (!socks || !errmsgs) - out_of_memory("open_socket_in"); /* We may not be able to create the socket, if for example the * machine knows about IPv6 in the C library, but not in the @@ -684,9 +680,6 @@ void set_socket_options(int fd, char *options) options = strdup(options); - if (!options) - out_of_memory("set_socket_options"); - for (tok = strtok(options, " \t,"); tok; tok = strtok(NULL," \t,")) { int ret=0,i; int value = 1; diff --git a/t_stub.c b/t_stub.c index 2748ca20f..5a9bad551 100644 --- a/t_stub.c +++ b/t_stub.c @@ -33,6 +33,7 @@ int preserve_xattrs = 0; int preserve_perms = 0; int preserve_executability = 0; int open_noatime = 0; +size_t max_alloc = 1024*1024*1024; char *partial_dir; char *module_dir; filter_rule_list daemon_filter_list; diff --git a/token.c b/token.c index 8a3358865..68d9bde3f 100644 --- a/token.c +++ b/token.c @@ -129,8 +129,7 @@ static void add_suffix(struct suffix_tree **prior, char ltr, const char *str) if (node->letter > ltr) break; } - if (!(newnode = new(struct suffix_tree))) - out_of_memory("add_suffix"); + newnode = new(struct suffix_tree); newnode->sibling = node; newnode->child = NULL; newnode->letter = ltr; @@ -147,8 +146,7 @@ static void add_nocompress_suffixes(const char *str) char *buf, *t; const char *f = str; - if (!(buf = new_array(char, strlen(f) + 1))) - out_of_memory("add_nocompress_suffixes"); + buf = new_array(char, strlen(f) + 1); while (*f) { if (*f == '/') { @@ -186,8 +184,7 @@ static void init_set_compression(void) else f = lp_dont_compress(module_id); - if (!(match_list = t = new_array(char, strlen(f) + 2))) - out_of_memory("set_compression"); + match_list = t = new_array(char, strlen(f) + 2); per_file_default_level = do_compression_level; @@ -282,11 +279,8 @@ static int32 simple_recv_token(int f, char **data) static char *buf; int32 n; - if (!buf) { + if (!buf) buf = new_array(char, CHUNK_SIZE); - if (!buf) - out_of_memory("simple_recv_token"); - } if (residue == 0) { int32 i = read_int(f); @@ -373,8 +367,7 @@ send_deflated_token(int f, int32 token, struct map_struct *buf, OFF_T offset, in rprintf(FERROR, "compression init failed\n"); exit_cleanup(RERR_PROTOCOL); } - if ((obuf = new_array(char, OBUF_SIZE)) == NULL) - out_of_memory("send_deflated_token"); + obuf = new_array(char, OBUF_SIZE); init_done = 1; } else deflateReset(&tx_strm); @@ -518,9 +511,8 @@ static int32 recv_deflated_token(int f, char **data) rprintf(FERROR, "inflate init failed\n"); exit_cleanup(RERR_PROTOCOL); } - if (!(cbuf = new_array(char, MAX_DATA_COUNT)) - || !(dbuf = new_array(char, AVAIL_OUT_SIZE(CHUNK_SIZE)))) - out_of_memory("recv_deflated_token"); + cbuf = new_array(char, MAX_DATA_COUNT); + dbuf = new_array(char, AVAIL_OUT_SIZE(CHUNK_SIZE)); init_done = 1; } else { inflateReset(&rx_strm); @@ -695,8 +687,6 @@ static void send_zstd_token(int f, int32 token, struct map_struct *buf, OFF_T of } obuf = new_array(char, OBUF_SIZE); - if (!obuf) - out_of_memory("send_deflated_token"); ZSTD_CCtx_setParameter(zstd_cctx, ZSTD_c_compressionLevel, do_compression_level); zstd_out_buff.dst = obuf + 2; @@ -806,8 +796,6 @@ static int32 recv_zstd_token(int f, char **data) out_buffer_size = ZSTD_DStreamOutSize() * 2; cbuf = new_array(char, MAX_DATA_COUNT); dbuf = new_array(char, out_buffer_size); - if (!cbuf || !dbuf) - out_of_memory("recv_zstd_token"); zstd_in_buff.src = cbuf; zstd_out_buff.dst = dbuf; @@ -903,8 +891,7 @@ send_compressed_token(int f, int32 token, struct map_struct *buf, OFF_T offset, if (last_token == -1) { if (!init_done) { - if ((obuf = new_array(char, size)) == NULL) - out_of_memory("send_compressed_token"); + obuf = new_array(char, size); init_done = 1; } last_run_end = 0; @@ -984,9 +971,8 @@ static int32 recv_compressed_token(int f, char **data) switch (recv_state) { case r_init: if (!init_done) { - if (!(cbuf = new_array(char, MAX_DATA_COUNT)) - || !(dbuf = new_array(char, size))) - out_of_memory("recv_compressed_token"); + cbuf = new_array(char, MAX_DATA_COUNT); + dbuf = new_array(char, size); init_done = 1; } recv_state = r_idle; diff --git a/uidlist.c b/uidlist.c index 739286a4a..7359d9dbc 100644 --- a/uidlist.c +++ b/uidlist.c @@ -85,8 +85,6 @@ static struct idlist *add_to_list(struct idlist **root, id_t id, union name_or_i id_t id2, uint16 flags) { struct idlist *node = new(struct idlist); - if (!node) - out_of_memory("add_to_list"); node->next = *root; node->u = noiu; node->id = id; @@ -160,8 +158,6 @@ static int is_in_group(gid_t gid) if ((ngroups = getgroups(0, NULL)) < 0) ngroups = 0; gidset = new_array(GETGROUPS_T, ngroups+1); - if (!gidset) - out_of_memory("is_in_group"); if (ngroups > 0) ngroups = getgroups(ngroups, gidset); /* The default gid might not be in the list on some systems. */ @@ -174,8 +170,6 @@ static int is_in_group(gid_t gid) if (DEBUG_GTE(OWN, 2)) { int pos; char *gidbuf = new_array(char, ngroups*21+32); - if (!gidbuf) - out_of_memory("is_in_group"); pos = snprintf(gidbuf, 32, "process has %d gid%s: ", ngroups, ngroups == 1? "" : "s"); for (n = 0; n < ngroups; n++) { pos += snprintf(gidbuf+pos, 21, " %d", (int)gidset[n]); @@ -375,8 +369,6 @@ uid_t recv_user_name(int f, uid_t uid) struct idlist *node; int len = read_byte(f); char *name = new_array(char, len+1); - if (!name) - out_of_memory("recv_user_name"); read_sbuf(f, name, len); if (numeric_ids < 0) { free(name); @@ -391,8 +383,6 @@ gid_t recv_group_name(int f, gid_t gid, uint16 *flags_ptr) struct idlist *node; int len = read_byte(f); char *name = new_array(char, len+1); - if (!name) - out_of_memory("recv_group_name"); read_sbuf(f, name, len); if (numeric_ids < 0) { free(name); diff --git a/util.c b/util.c index 6489e650d..3a50e44f5 100644 --- a/util.c +++ b/util.c @@ -592,8 +592,7 @@ int lock_range(int fd, int offset, int len) } #define ENSURE_MEMSPACE(buf, type, sz, req) \ - if ((req) > sz && !(buf = realloc_array(buf, type, sz = MAX(sz * 2, req)))) \ - out_of_memory("glob_expand") + do { if ((req) > sz) buf = realloc_array(buf, type, sz = MAX(sz * 2, req)); } while(0) static inline void call_glob_match(const char *name, int len, int from_glob, char *arg, int abpos, int fbpos); @@ -695,8 +694,7 @@ static inline void call_glob_match(const char *name, int len, int from_glob, glob_match(arg, abpos, fbpos); } else { ENSURE_MEMSPACE(glob.argv, char *, glob.maxargs, glob.argc + 1); - if (!(glob.argv[glob.argc++] = strdup(glob.arg_buf))) - out_of_memory("glob_match"); + glob.argv[glob.argc++] = strdup(glob.arg_buf); } } @@ -720,8 +718,6 @@ int glob_expand(const char *arg, char ***argv_p, int *argc_p, int *maxargs_p) s = sanitize_path(NULL, arg, "", 0, SP_KEEP_DOT_DIRS); else { s = strdup(arg); - if (!s) - out_of_memory("glob_expand"); clean_fname(s, CFN_KEEP_DOT_DIRS | CFN_KEEP_TRAILING_SLASH | CFN_COLLAPSE_DOT_DOT_DIRS); } @@ -771,8 +767,7 @@ void glob_expand_module(char *base1, char *arg, char ***argv_p, int *argc_p, int return; } - if (!(arg = strdup(arg))) - out_of_memory("glob_expand_module"); + arg = strdup(arg); if (asprintf(&base," %s/", base1) < 0) out_of_memory("glob_expand_module"); @@ -1017,11 +1012,10 @@ char *sanitize_path(char *dest, const char *p, const char *rootdir, int depth, i depth = 0; p++; } - if (dest) { - if (rlen + plen + 1 >= MAXPATHLEN) - return NULL; - } else if (!(dest = new_array(char, MAX(rlen + plen + 1, 2)))) - out_of_memory("sanitize_path"); + if (!dest) + dest = new_array(char, MAX(rlen + plen + 1, 2)); + else if (rlen + plen + 1 >= MAXPATHLEN) + return NULL; if (rlen) { /* only true if p previously started with a slash */ memcpy(dest, rootdir, rlen); if (rlen > 1) /* a rootdir of len 1 is "/", so this avoids a 2nd slash */ @@ -1155,13 +1149,10 @@ char *normalize_path(char *path, BOOL force_newbuf, unsigned int *len_ptr) return NULL; curr_dir[curr_dir_len] = '/'; memcpy(curr_dir + curr_dir_len + 1, path, len + 1); - if (!(path = strdup(curr_dir))) - out_of_memory("normalize_path"); + path = strdup(curr_dir); curr_dir[curr_dir_len] = '\0'; - } else if (force_newbuf) { - if (!(path = strdup(path))) - out_of_memory("normalize_path"); - } + } else if (force_newbuf) + path = strdup(path); len = clean_fname(path, CFN_COLLAPSE_DOT_DOT_DIRS | CFN_DROP_TRAILING_DOT_DIR); @@ -1519,8 +1510,7 @@ struct bitbag *bitbag_create(int max_ndx) struct bitbag *bb = new(struct bitbag); bb->slot_cnt = (max_ndx + BB_PER_SLOT_BITS - 1) / BB_PER_SLOT_BITS; - if (!(bb->bits = (uint32**)calloc(bb->slot_cnt, sizeof (uint32*)))) - out_of_memory("bitbag_create"); + bb->bits = (uint32**)calloc(bb->slot_cnt, sizeof (uint32*)); return bb; } @@ -1601,8 +1591,7 @@ void flist_ndx_push(flist_ndx_list *lp, int ndx) { struct flist_ndx_item *item; - if (!(item = new(struct flist_ndx_item))) - out_of_memory("flist_ndx_push"); + item = new(struct flist_ndx_item); item->next = NULL; item->ndx = ndx; if (lp->tail) @@ -1654,8 +1643,7 @@ void *expand_item_list(item_list *lp, size_t item_size, const char *desc, int in new_size = 1; if (new_size <= lp->malloced) overflow_exit("expand_item_list"); - /* Using _realloc_array() lets us pass the size, not a type. */ - new_ptr = _realloc_array(lp->items, new_size, item_size); + new_ptr = realloc_buf(lp->items, new_size * item_size); if (DEBUG_GTE(FLIST, 3)) { rprintf(FINFO, "[%s] expand %s to %s bytes, did%s move\n", who_am_i(), desc, big_num(new_size * item_size), diff --git a/util2.c b/util2.c index 89e2f3756..10b7e4b6a 100644 --- a/util2.c +++ b/util2.c @@ -25,6 +25,10 @@ #include "itypes.h" #include "inums.h" +extern size_t max_alloc; + +char *do_malloc = "42"; + /** * Sleep for a specified number of milliseconds. * @@ -67,22 +71,39 @@ int msleep(int t) return True; } -#define MALLOC_MAX 0x40000000 - -void *_new_array(size_t num, size_t size, int use_calloc) +/* We convert a num manually because need %lld precision, and that's not a portable sprintf() escape. */ +char *num_to_byte_string(ssize_t num) { - if (num >= MALLOC_MAX/size) - return NULL; - return use_calloc ? calloc(num, size) : malloc(num * size); + char buf[128], *s = buf + sizeof buf - 1; + + *s = '\0'; + while (num) { + *--s = (char)(num % 10) + '0'; + num /= 10; + } + return strdup(s); } -void *_realloc_array(void *ptr, size_t num, size_t size) +void *_my_alloc(void *ptr, size_t num, size_t size, const char *file, int line) { - if (num >= MALLOC_MAX/size) - return NULL; + if (num >= max_alloc/size) { + if (!file) + return NULL; + rprintf(FERROR, "[%s] exceeded --max-alloc=%s setting (file=%s, line=%d)\n", + who_am_i(), num_to_byte_string(max_alloc), file, line); + exit_cleanup(RERR_MALLOC); + } if (!ptr) - return malloc(size * num); - return realloc(ptr, size * num); + ptr = calloc(num, size); + else if (ptr == do_malloc) + ptr = malloc(num * size); + else + ptr = realloc(ptr, num * size); + if (!ptr && file) { + rprintf(FERROR, "[%s] out of memory (file=%s, line=%d)\n", who_am_i(), file, line); + exit_cleanup(RERR_MALLOC); + } + return ptr; } const char *sum_as_hex(int csum_type, const char *sum, int flist_csum) diff --git a/xattrs.c b/xattrs.c index bbe003b97..b3f0c1a34 100644 --- a/xattrs.c +++ b/xattrs.c @@ -145,8 +145,6 @@ static ssize_t get_xattr_names(const char *fname) if (!namebuf) { namebuf_len = 1024; namebuf = new_array(char, namebuf_len); - if (!namebuf) - out_of_memory("get_xattr_names"); } while (1) { @@ -174,8 +172,6 @@ static ssize_t get_xattr_names(const char *fname) free(namebuf); namebuf_len = list_len + 1024; namebuf = new_array(char, namebuf_len); - if (!namebuf) - out_of_memory("get_xattr_names"); } return list_len; @@ -205,8 +201,7 @@ static char *get_xattr_data(const char *fname, const char *name, size_t *len_ptr extra_len = 1; /* request non-zero amount of memory */ if (datum_len + extra_len < datum_len) overflow_exit("get_xattr_data"); - if (!(ptr = new_array(char, datum_len + extra_len))) - out_of_memory("get_xattr_data"); + ptr = new_array(char, datum_len + extra_len); if (datum_len) { size_t len = sys_lgetxattr(fname, name, ptr, datum_len); @@ -279,8 +274,7 @@ static int rsync_xal_get(const char *fname, item_list *xalp) sum_update(ptr, datum_len); free(ptr); - if (!(ptr = new_array(char, name_offset + name_len))) - out_of_memory("rsync_xal_get"); + ptr = new_array(char, name_offset + name_len); *ptr = XSTATE_ABBREV; sum_end(ptr + 1); } else @@ -481,8 +475,6 @@ static int rsync_xal_store(item_list *xalp) out_of_memory("rsync_xal_h hashtable_create()"); new_ref = new0(rsync_xa_list_ref); - if (new_ref == NULL) - out_of_memory("new0(rsync_xa_list_ref)"); new_ref->ndx = ndx; node = hashtable_find(rsync_xal_h, new_list->key, new_ref); @@ -759,8 +751,6 @@ int recv_xattr_request(struct file_struct *file, int f_in) if (rxa->name_len + rxa->datum_len < rxa->name_len) overflow_exit("recv_xattr_request"); rxa->datum = new_array(char, rxa->datum_len + rxa->name_len); - if (!rxa->datum) - out_of_memory("recv_xattr_request"); name = rxa->datum + rxa->datum_len; memcpy(name, rxa->name, rxa->name_len); rxa->name = name; @@ -813,8 +803,6 @@ void receive_xattr(int f, struct file_struct *file) || (dget_len + extra_len + name_len < dget_len + extra_len)) overflow_exit("receive_xattr"); ptr = new_array(char, dget_len + extra_len + name_len); - if (!ptr) - out_of_memory("receive_xattr"); name = ptr + dget_len + extra_len; read_buf(f, name, name_len); if (name_len < 1 || name[name_len-1] != '\0') {