The --iconv option has now made it to the trunk.

NEWS

@@ -61,6 +61,10 @@ Changes since 2.6.9:
       all attributes of a file by using a special extended-attribute idiom.
       There is also an analogous "fake super" option for an rsync daemon.
 
+    - Added the --iconv option, which allows rsync to convert filenames from
+      one character set to another during the transfer.  (Currently must be
+      manually enabled via configure's --enable-iconv option.)
+
     - You may specify --max-delete=0 to a 3.0.0 client as long as the receiving
       side is at least version 3.0.0.  This means that you can pull from an
       older rsync with this option, but pushing to an older rsync will generate

clientserver.c

@@ -687,7 +687,7 @@ static int rsync_module(int f_in, int f_out, int i, char *addr, char *host)
 
 	if (!ret || err_msg) {
 		if (err_msg)
-			rwrite(FERROR, err_msg, strlen(err_msg));
+			rwrite(FERROR, err_msg, strlen(err_msg), 0);
 		else
 			option_error();
 		msleep(400);

configure.in

@@ -568,6 +568,22 @@ if test $ac_cv_func_getpgrp = yes; then
     AC_FUNC_GETPGRP
 fi
 
+AC_ARG_ENABLE(iconv,
+    AC_HELP_STRING([--disable-iconv],
+	    [disable rsync's --iconv option]),
+    [], [enable_iconv=$ac_cv_func_iconv_open])
+AH_TEMPLATE([ICONV_OPTION],
+[Define if you want the --iconv option.  Specifing a value will set the
+default iconv setting (a NULL means no --iconv processing by default).])
+if test x"$enable_iconv" != x"no"; then
+	if test x"$enable_iconv" = x"yes"; then
+		AC_DEFINE(ICONV_OPTION, NULL)
+	else
+		AC_DEFINE_UNQUOTED(ICONV_OPTION, "$enable_iconv")
+	fi
+	AC_DEFINE(UTF8_CHARSET, "UTF-8", [String to pass to iconv() for the UTF-8 charset.])
+fi
+
 AC_CACHE_CHECK([whether chown() modifies symlinks],rsync_cv_chown_modifies_symlink,[
   AC_TRY_RUN([
 #if HAVE_UNISTD_H

flist.c

@@ -70,6 +70,12 @@ extern struct chmod_mode_struct *chmod_modes;
 extern struct filter_list_struct filter_list;
 extern struct filter_list_struct server_filter_list;
 
+#ifdef ICONV_OPTION
+extern int ic_ndx;
+extern int need_unsorted_flist;
+extern iconv_t ic_send, ic_recv;
+#endif
+
 int io_error;
 int checksum_len;
 dev_t filesystem_dev; /* used to implement -x */
@@ -346,7 +352,35 @@ static void send_file_entry(int f, struct file_struct *file, int ndx)
 	int l1, l2;
 	int flags;
 
-	f_name(file, fname);
+#ifdef ICONV_OPTION
+	if (ic_send != (iconv_t)-1) {
+		ICONV_CONST char *ibuf;
+		char *obuf = fname;
+		size_t ocnt = MAXPATHLEN, icnt;
+
+		iconv(ic_send, NULL,0, NULL,0);
+		if ((ibuf = (ICONV_CONST char *)file->dirname) != NULL) {
+		    icnt = strlen(ibuf);
+		    ocnt--; /* pre-subtract the space for the '/' */
+		    if (iconv(ic_send, &ibuf,&icnt, &obuf,&ocnt) == (size_t)-1)
+			goto convert_error;
+		    *obuf++ = '/';
+		}
+
+		ibuf = (ICONV_CONST char *)file->basename;
+		icnt = strlen(ibuf);
+		if (iconv(ic_send, &ibuf,&icnt, &obuf,&ocnt) == (size_t)-1) {
+		  convert_error:
+			io_error |= IOERR_GENERAL;
+			rprintf(FINFO,
+			    "[%s] cannot convert filename: %s (%s)\n",
+			    who_am_i(), f_name(file, fname), strerror(errno));
+			return;
+		}
+		*obuf = '\0';
+	} else
+#endif
+		f_name(file, fname);
 
 	flags = file->flags & FLAG_TOP_DIR; /* FLAG_TOP_DIR == XMIT_TOP_DIR */
 
@@ -598,7 +632,31 @@ static struct file_struct *recv_file_entry(struct file_list *flist,
 	read_sbuf(f, &thisname[l1], l2);
 	thisname[l1 + l2] = 0;
 
-	strlcpy(lastname, thisname, MAXPATHLEN);
+	/* Abuse basename_len for a moment... */
+	basename_len = strlcpy(lastname, thisname, MAXPATHLEN);
+
+#ifdef ICONV_OPTION
+	if (ic_recv != (iconv_t)-1) {
+		char *obuf = thisname, *ibuf = lastname;
+		size_t ocnt = MAXPATHLEN, icnt = basename_len;
+
+		if (icnt >= MAXPATHLEN) {
+			errno = E2BIG;
+			goto convert_error;
+		}
+
+		iconv(ic_recv, NULL,0, NULL,0);
+		if (iconv(ic_recv, &ibuf,&icnt, &obuf,&ocnt) == (size_t)-1) {
+		  convert_error:
+			io_error |= IOERR_GENERAL;
+			rprintf(FINFO,
+			    "[%s] cannot convert filename: %s (%s)\n",
+			    who_am_i(), lastname, strerror(errno));
+			obuf = thisname;
+		}
+		*obuf = '\0';
+	}
+#endif
 
 	clean_fname(thisname, 0);
 
@@ -792,6 +850,10 @@ static struct file_struct *recv_file_entry(struct file_list *flist,
 		F_OWNER(file) = uid;
 	if (preserve_gid)
 		F_GROUP(file) = gid;
+#ifdef ICONV_OPTION
+	if (ic_ndx)
+		F_NDX(file) = flist->count + flist->ndx_start;
+#endif
 
 	if (basename != thisname) {
 		file->dirname = lastdir;
@@ -1385,7 +1447,13 @@ void send_extra_file_list(int f, int at_least)
 			       FLAG_DIVERT_DIRS | FLAG_XFER_DIR);
 		write_byte(f, 0);
 
-		clean_flist(flist, 0, 0);
+#ifdef ICONV_OPTION
+		if (!need_unsorted_flist)
+#endif
+		{
+			flist->sorted = flist->files;
+			clean_flist(flist, 0, 0);
+		}
 		file_total += flist->count;
 		future_cnt += flist->count;
 		stats.flist_size += stats.total_written - start_write;
@@ -1670,12 +1738,30 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
 	stats.flist_xfertime = (int64)(end_tv.tv_sec - start_tv.tv_sec) * 1000
 			     + (end_tv.tv_usec - start_tv.tv_usec) / 1000;
 
+	/* When converting names, both sides keep an unsorted file-list array
+	 * because the names will differ on the sending and receiving sides
+	 * (both sides will use the unsorted index number for each item). */
+
 	/* Sort the list without removing any duplicates in non-incremental
 	 * mode.  This allows the receiving side to ask for whatever name it
 	 * kept.  For incremental mode, the sender also removes duplicates
 	 * in this initial file-list so that it avoids re-sending duplicated
 	 * directories. */
-	clean_flist(flist, 0, inc_recurse);
+#ifdef ICONV_OPTION
+	if (need_unsorted_flist) {
+		if (inc_recurse) {
+			if (!(flist->sorted = new_array(struct file_struct *, flist->count)))
+				out_of_memory("recv_file_list");
+			memcpy(flist->sorted, flist->files,
+			       flist->count * sizeof (struct file_struct*));
+			clean_flist(flist, 0, 1);
+		}
+	} else
+#endif
+	{
+		flist->sorted = flist->files;
+		clean_flist(flist, 0, inc_recurse);
+	}
 	file_total += flist->count;
 
 	if (!numeric_ids && !inc_recurse)
@@ -1778,11 +1864,27 @@ struct file_list *recv_file_list(int f)
 	if (show_filelist_p())
 		finish_filelist_progress(flist);
 
+#ifdef ICONV_OPTION
+	if (need_unsorted_flist) {
+		/* Create an extra array of index pointers that we can sort for
+		 * the generator's use (for wading through the files in sorted
+		 * order and for calling flist_find()).  We keep the "files"
+		 * list unsorted for our exchange of index numbers with the
+		 * other side (since their names may not sort the same). */
+		if (!(flist->sorted = new_array(struct file_struct *, flist->count)))
+			out_of_memory("recv_file_list");
+		memcpy(flist->sorted, flist->files,
+		       flist->count * sizeof (struct file_struct*));
+	} else
+#endif
+		flist->sorted = flist->files;
+
 	clean_flist(flist, relative_paths, 1);
 
 	if (inc_recurse) {
-		qsort(dir_flist->files + dstart, dir_flist->count - dstart,
+		dir_flist->sorted = dir_flist->files;
-		      sizeof dir_flist->files[0], (int (*)())file_compare);
+		qsort(dir_flist->sorted + dstart, dir_flist->count - dstart,
+		      sizeof dir_flist->sorted[0], (int (*)())file_compare);
 	} else if (f >= 0)
 		recv_uid_list(f, flist);
 
@@ -1845,36 +1947,36 @@ int flist_find(struct file_list *flist, struct file_struct *f)
 
 	while (low <= high) {
 		mid = (low + high) / 2;
-		if (F_IS_ACTIVE(flist->files[mid]))
+		if (F_IS_ACTIVE(flist->sorted[mid]))
 			mid_up = mid;
 		else {
 			/* Scan for the next non-empty entry using the cached
 			 * distance values.  If the value isn't fully up-to-
 			 * date, update it. */
-			mid_up = mid + F_DEPTH(flist->files[mid]);
+			mid_up = mid + F_DEPTH(flist->sorted[mid]);
-			if (!F_IS_ACTIVE(flist->files[mid_up])) {
+			if (!F_IS_ACTIVE(flist->sorted[mid_up])) {
 				do {
-				    mid_up += F_DEPTH(flist->files[mid_up]);
+				    mid_up += F_DEPTH(flist->sorted[mid_up]);
-				} while (!F_IS_ACTIVE(flist->files[mid_up]));
+				} while (!F_IS_ACTIVE(flist->sorted[mid_up]));
-				F_DEPTH(flist->files[mid]) = mid_up - mid;
+				F_DEPTH(flist->sorted[mid]) = mid_up - mid;
 			}
 			if (mid_up > high) {
 				/* If there's nothing left above us, set high to
 				 * a non-empty entry below us and continue. */
-				high = mid - (int)flist->files[mid]->len32;
+				high = mid - (int)flist->sorted[mid]->len32;
-				if (!F_IS_ACTIVE(flist->files[high])) {
+				if (!F_IS_ACTIVE(flist->sorted[high])) {
 					do {
-					    high -= (int)flist->files[high]->len32;
+					    high -= (int)flist->sorted[high]->len32;
-					} while (!F_IS_ACTIVE(flist->files[high]));
+					} while (!F_IS_ACTIVE(flist->sorted[high]));
-					flist->files[mid]->len32 = mid - high;
+					flist->sorted[mid]->len32 = mid - high;
 				}
 				continue;
 			}
 		}
-		diff = f_name_cmp(flist->files[mid_up], f);
+		diff = f_name_cmp(flist->sorted[mid_up], f);
 		if (diff == 0) {
 			if (protocol_version < 29
-			    && S_ISDIR(flist->files[mid_up]->mode)
+			    && S_ISDIR(flist->sorted[mid_up]->mode)
 			    != S_ISDIR(f->mode))
 				return -1;
 			return mid_up;
@@ -1960,6 +2062,8 @@ void flist_free(struct file_list *flist)
 	}
 
 	pool_destroy(flist->file_pool);
+	if (flist->sorted && flist->sorted != flist->files)
+		free(flist->sorted);
 	free(flist->files);
 	free(flist);
 }
@@ -1980,11 +2084,11 @@ static void clean_flist(struct file_list *flist, int strip_root, int no_dups)
 		return;
 	}
 
-	qsort(flist->files, flist->count,
+	qsort(flist->sorted, flist->count,
-	    sizeof flist->files[0], (int (*)())file_compare);
+	    sizeof flist->sorted[0], (int (*)())file_compare);
 
 	for (i = no_dups? 0 : flist->count; i < flist->count; i++) {
-		if (F_IS_ACTIVE(flist->files[i])) {
+		if (F_IS_ACTIVE(flist->sorted[i])) {
 			prev_i = i;
 			break;
 		}
@@ -1992,11 +2096,11 @@ static void clean_flist(struct file_list *flist, int strip_root, int no_dups)
 	flist->low = prev_i;
 	while (++i < flist->count) {
 		int j;
-		struct file_struct *file = flist->files[i];
+		struct file_struct *file = flist->sorted[i];
 
 		if (!F_IS_ACTIVE(file))
 			continue;
-		if (f_name_cmp(file, flist->files[prev_i]) == 0)
+		if (f_name_cmp(file, flist->sorted[prev_i]) == 0)
 			j = prev_i;
 		else if (protocol_version >= 29 && S_ISDIR(file->mode)) {
 			int save_mode = file->mode;
@@ -2009,7 +2113,7 @@ static void clean_flist(struct file_list *flist, int strip_root, int no_dups)
 		} else
 			j = -1;
 		if (j >= 0) {
-			struct file_struct *fp = flist->files[j];
+			struct file_struct *fp = flist->sorted[j];
 			int keep, drop;
 			/* If one is a dir and the other is not, we want to
 			 * keep the dir because it might have contents in the
@@ -2030,15 +2134,15 @@ static void clean_flist(struct file_list *flist, int strip_root, int no_dups)
 			}
 			/* Make sure we don't lose track of a user-specified
 			 * top directory. */
-			flist->files[keep]->flags |= flist->files[drop]->flags
+			flist->sorted[keep]->flags |= flist->sorted[drop]->flags
 						   & (FLAG_TOP_DIR|FLAG_XFER_DIR);
 
-			clear_file(flist->files[drop]);
+			clear_file(flist->sorted[drop]);
 
 			if (keep == i) {
 				if (flist->low == drop) {
 					for (j = drop + 1;
-					     j < i && !F_IS_ACTIVE(flist->files[j]);
+					     j < i && !F_IS_ACTIVE(flist->sorted[j]);
 					     j++) {}
 					flist->low = j;
 				}
@@ -2053,7 +2157,7 @@ static void clean_flist(struct file_list *flist, int strip_root, int no_dups)
 		/* We need to strip off the leading slashes for relative
 		 * paths, but this must be done _after_ the sorting phase. */
 		for (i = flist->low; i <= flist->high; i++) {
-			struct file_struct *file = flist->files[i];
+			struct file_struct *file = flist->sorted[i];
 
 			if (!file->dirname)
 				continue;
@@ -2070,7 +2174,7 @@ static void clean_flist(struct file_list *flist, int strip_root, int no_dups)
 		prev_i = 0; /* It's OK that this isn't really true. */
 
 		for (i = flist->low; i <= flist->high; i++) {
-			struct file_struct *fp, *file = flist->files[i];
+			struct file_struct *fp, *file = flist->sorted[i];
 
 			/* This temporarily abuses the F_DEPTH() value for a
 			 * directory that is in a chain that might get pruned.
@@ -2078,7 +2182,7 @@ static void clean_flist(struct file_list *flist, int strip_root, int no_dups)
 			if (S_ISDIR(file->mode) && F_DEPTH(file)) {
 				/* Dump empty dirs when coming back down. */
 				for (j = prev_depth; j >= F_DEPTH(file); j--) {
-					fp = flist->files[prev_i];
+					fp = flist->sorted[prev_i];
 					if (F_DEPTH(fp) >= 0)
 						break;
 					prev_i = -F_DEPTH(fp)-1;
@@ -2089,7 +2193,7 @@ static void clean_flist(struct file_list *flist, int strip_root, int no_dups)
 						       ALL_FILTERS)) {
 					/* Keep dirs through this dir. */
 					for (j = prev_depth-1; ; j--) {
-						fp = flist->files[prev_i];
+						fp = flist->sorted[prev_i];
 						if (F_DEPTH(fp) >= 0)
 							break;
 						prev_i = -F_DEPTH(fp)-1;
@@ -2101,7 +2205,7 @@ static void clean_flist(struct file_list *flist, int strip_root, int no_dups)
 			} else {
 				/* Keep dirs through this non-dir. */
 				for (j = prev_depth; ; j--) {
-					fp = flist->files[prev_i];
+					fp = flist->sorted[prev_i];
 					if (F_DEPTH(fp) >= 0)
 						break;
 					prev_i = -F_DEPTH(fp)-1;
@@ -2111,7 +2215,7 @@ static void clean_flist(struct file_list *flist, int strip_root, int no_dups)
 		}
 		/* Dump all remaining empty dirs. */
 		while (1) {
-			struct file_struct *fp = flist->files[prev_i];
+			struct file_struct *fp = flist->sorted[prev_i];
 			if (F_DEPTH(fp) >= 0)
 				break;
 			prev_i = -F_DEPTH(fp)-1;
@@ -2119,12 +2223,12 @@ static void clean_flist(struct file_list *flist, int strip_root, int no_dups)
 		}
 
 		for (i = flist->low; i <= flist->high; i++) {
-			if (F_IS_ACTIVE(flist->files[i]))
+			if (F_IS_ACTIVE(flist->sorted[i]))
 				break;
 		}
 		flist->low = i;
 		for (i = flist->high; i >= flist->low; i--) {
-			if (F_IS_ACTIVE(flist->files[i]))
+			if (F_IS_ACTIVE(flist->sorted[i]))
 				break;
 		}
 		flist->high = i;
@@ -2369,6 +2473,7 @@ struct file_list *get_dirlist(char *dirname, int dlen, int ignore_filter_rules)
 	if (do_progress)
 		flist_count_offset += dirlist->count;
 
+	dirlist->sorted = dirlist->files;
 	clean_flist(dirlist, 0, 0);
 
 	if (verbose > 3)

generator.c

@@ -97,6 +97,9 @@ extern char *backup_suffix;
 extern int backup_suffix_len;
 extern struct file_list *cur_flist, *first_flist, *dir_flist;
 extern struct filter_list_struct server_filter_list;
+#ifdef ICONV_OPTION
+extern int ic_ndx;
+#endif
 
 int ignore_perishable = 0;
 int non_perishable_cnt = 0;
@@ -415,15 +418,14 @@ static void do_delayed_deletions(char *delbuf)
  * MAXPATHLEN buffer with the name of the directory in it (the functions we
  * call will append names onto the end, but the old dir value will be restored
  * on exit). */
-static void delete_in_dir(struct file_list *flist, char *fbuf,
+static void delete_in_dir(char *fbuf, struct file_struct *file, dev_t *fs_dev)
-			  struct file_struct *file, dev_t *fs_dev)
 {
 	static int already_warned = 0;
 	struct file_list *dirlist;
 	char delbuf[MAXPATHLEN];
 	int dlen, i;
 
-	if (!flist) {
+	if (!fbuf) {
 		change_local_filter_dir(NULL, 0, 0);
 		return;
 	}
@@ -467,7 +469,7 @@ static void delete_in_dir(struct file_list *flist, char *fbuf,
 					f_name(fp, NULL));
 			continue;
 		}
-		if (flist_find(flist, fp) < 0) {
+		if (flist_find(cur_flist, fp) < 0) {
 			f_name(fp, delbuf);
 			if (delete_during == 2) {
 				if (!remember_delete(fp, delbuf))
@@ -482,7 +484,7 @@ static void delete_in_dir(struct file_list *flist, char *fbuf,
 
 /* This deletes any files on the receiving side that are not present on the
  * sending side.  This is used by --delete-before and --delete-after. */
-static void do_delete_pass(struct file_list *flist)
+static void do_delete_pass(void)
 {
 	char fbuf[MAXPATHLEN];
 	STRUCT_STAT st;
@@ -492,8 +494,8 @@ static void do_delete_pass(struct file_list *flist)
 	if (dry_run > 1 || list_only)
 		return;
 
-	for (j = 0; j < flist->count; j++) {
+	for (j = 0; j < cur_flist->count; j++) {
-		struct file_struct *file = flist->files[j];
+		struct file_struct *file = cur_flist->sorted[j];
 
 		if (!(file->flags & FLAG_XFER_DIR))
 			continue;
@@ -506,9 +508,9 @@ static void do_delete_pass(struct file_list *flist)
 		 || !S_ISDIR(st.st_mode))
 			continue;
 
-		delete_in_dir(flist, fbuf, file, &st.st_dev);
+		delete_in_dir(fbuf, file, &st.st_dev);
 	}
-	delete_in_dir(NULL, NULL, NULL, &dev_zero);
+	delete_in_dir(NULL, NULL, &dev_zero);
 
 	if (do_progress && !am_server)
 		rprintf(FINFO, "                    \r");
@@ -1271,7 +1273,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx,
 		}
 		else if (delete_during && f_out != -1 && !phase && dry_run < 2
 		    && (file->flags & FLAG_XFER_DIR))
-			delete_in_dir(cur_flist, fname, file, &real_sx.st.st_dev);
+			delete_in_dir(fname, file, &real_sx.st.st_dev);
 		goto cleanup;
 	}
 
@@ -1731,7 +1733,7 @@ static void touch_up_dirs(struct file_list *flist, int ndx)
 	/* Fix any directory permissions that were modified during the
 	 * transfer and/or re-set any tweaked modified-time values. */
 	for (i = start, j = 0; i <= end; i++) {
-		file = flist->files[i];
+		file = flist->sorted[i];
 		if (!F_IS_ACTIVE(file) || !S_ISDIR(file->mode)
 		 || file->flags & FLAG_MISSING_DIR)
 			continue;
@@ -1827,7 +1829,7 @@ void check_for_finished_files(int itemizing, enum logcode code, int check_redo)
 
 void generate_files(int f_out, const char *local_name)
 {
-	int i;
+	int i, ndx;
 	char fbuf[MAXPATHLEN];
 	int itemizing;
 	enum logcode code;
@@ -1860,7 +1862,7 @@ void generate_files(int f_out, const char *local_name)
 		rprintf(FINFO, "generator starting pid=%ld\n", (long)getpid());
 
 	if (delete_before && !solo_file && cur_flist->count > 0)
-		do_delete_pass(cur_flist);
+		do_delete_pass();
 	if (delete_during == 2) {
 		deldelay_size = BIGPATHBUFLEN * 4;
 		deldelay_buf = new_array(char, deldelay_size);
@@ -1896,21 +1898,27 @@ void generate_files(int f_out, const char *local_name)
 					dirdev = MAKEDEV(DEV_MAJOR(devp), DEV_MINOR(devp));
 				} else
 					dirdev = MAKEDEV(0, 0);
-				delete_in_dir(cur_flist, f_name(fp, fbuf), fp, &dirdev);
+				delete_in_dir(f_name(fp, fbuf), fp, &dirdev);
 			}
 		}
 		for (i = cur_flist->low; i <= cur_flist->high; i++) {
-			struct file_struct *file = cur_flist->files[i];
+			struct file_struct *file = cur_flist->sorted[i];
 
 			if (!F_IS_ACTIVE(file))
 				continue;
 
+#ifdef ICONV_OPTION
+			if (ic_ndx)
+				ndx = F_NDX(file);
+			else
+#endif
+				ndx = i + cur_flist->ndx_start;
+
 			if (solo_file)
 				strlcpy(fbuf, solo_file, sizeof fbuf);
 			else
 				f_name(file, fbuf);
-			recv_generator(fbuf, file, i + cur_flist->ndx_start,
+			recv_generator(fbuf, file, ndx, itemizing, code, f_out);
-				       itemizing, code, f_out);
 
 			/* We need to ensure that any dirs we create have
 			 * writeable permissions during the time we are putting
@@ -1952,7 +1960,7 @@ void generate_files(int f_out, const char *local_name)
 	} while ((cur_flist = cur_flist->next) != NULL);
 
 	if (delete_during)
-		delete_in_dir(NULL, NULL, NULL, &dev_zero);
+		delete_in_dir(NULL, NULL, &dev_zero);
 	phase++;
 	if (verbose > 2)
 		rprintf(FINFO, "generate_files phase=%d\n", phase);
@@ -1996,7 +2004,7 @@ void generate_files(int f_out, const char *local_name)
 	if (delete_during == 2)
 		do_delayed_deletions(fbuf);
 	if (delete_after && !solo_file && file_total > 0)
-		do_delete_pass(cur_flist);
+		do_delete_pass();
 
 	if ((need_retouch_dir_perms || need_retouch_dir_times)
 	 && dir_tweaking && (!inc_recurse || delete_during == 2))

hlink.c

@@ -34,11 +34,13 @@ extern int stdout_format_has_i;
 extern int maybe_ATTRS_REPORT;
 extern char *basis_dir[];
 extern struct file_list *cur_flist;
+#ifdef ICONV_OPTION
+extern int ic_ndx;
+#endif
 
 #ifdef SUPPORT_HARD_LINKS
 
 #define HASH_LOAD_LIMIT(size) ((size)*3/4)
-#define FPTR(i) (cur_flist->files[i])
 
 struct ihash_table {
 	int32 size;
@@ -200,8 +202,8 @@ void idev_destroy(void)
 
 static int hlink_compare_gnum(int *int1, int *int2)
 {
-	struct file_struct *f1 = FPTR(*int1);
+	struct file_struct *f1 = cur_flist->sorted[*int1];
-	struct file_struct *f2 = FPTR(*int2);
+	struct file_struct *f2 = cur_flist->sorted[*int2];
 	int32 gnum1 = F_HL_GNUM(f1);
 	int32 gnum2 = F_HL_GNUM(f2);
 
@@ -221,17 +223,24 @@ static void match_gnums(int32 *ndx_list, int ndx_count)
 	     (int (*)()) hlink_compare_gnum);
 
 	for (from = 0; from < ndx_count; from++) {
-		for (file = FPTR(ndx_list[from]), gnum = F_HL_GNUM(file), prev = -1;
+		for (file = cur_flist->sorted[ndx_list[from]], gnum = F_HL_GNUM(file), prev = -1;
 		     from < ndx_count-1;
-		     file = file_next, gnum = gnum_next, prev = ndx_list[from++])
+		     file = file_next, gnum = gnum_next, from++)
 		{
-			file_next = FPTR(ndx_list[from+1]);
+			file_next = cur_flist->sorted[ndx_list[from+1]];
 			gnum_next = F_HL_GNUM(file_next);
 			if (gnum != gnum_next)
 				break;
 			if (prev < 0)
 				file->flags |= FLAG_HLINK_FIRST;
 			F_HL_PREV(file) = prev;
+			/* The linked list must use raw ndx values. */
+#ifdef ICONV_OPTION
+			if (ic_ndx)
+				prev = F_NDX(file);
+			else
+#endif
+				prev = ndx_list[from];
 		}
 		if (prev < 0)
 			file->flags &= ~FLAG_HLINKED;
@@ -255,7 +264,7 @@ void match_hard_links(void)
 		out_of_memory("match_hard_links");
 
 	for (i = 0; i < cur_flist->count; i++) {
-		if (F_IS_HLINKED(FPTR(i)))
+		if (F_IS_HLINKED(cur_flist->sorted[i]))
 			ndx_list[ndx_count++] = i;
 	}
 
@@ -317,7 +326,7 @@ int hard_link_check(struct file_struct *file, int ndx, const char *fname,
 	STRUCT_STAT prev_st;
 	char prev_name[MAXPATHLEN], altbuf[MAXPATHLEN], *realname;
 	int alt_dest, prev_ndx = F_HL_PREV(file);
-	struct file_struct *prev_file = FPTR(prev_ndx);
+	struct file_struct *prev_file = cur_flist->files[prev_ndx];
 
 	/* Is the previous link is not complete yet? */
 	if (!(prev_file->flags & FLAG_HLINK_DONE)) {
@@ -338,7 +347,7 @@ int hard_link_check(struct file_struct *file, int ndx, const char *fname,
 	if (!(prev_file->flags & FLAG_HLINK_FIRST)) {
 		/* The previous previous will be marked with FIRST. */
 		prev_ndx = F_HL_PREV(prev_file);
-		prev_file = FPTR(prev_ndx);
+		prev_file = cur_flist->files[prev_ndx];
 		/* Update our previous pointer to point to the first. */
 		F_HL_PREV(file) = prev_ndx;
 	}
@@ -475,7 +484,7 @@ void finish_hard_link(struct file_struct *file, const char *fname,
 
 	while ((ndx = prev_ndx) >= 0) {
 		int val;
-		file = FPTR(ndx);
+		file = cur_flist->files[ndx];
 		file->flags = (file->flags & ~FLAG_HLINK_FIRST) | FLAG_HLINK_DONE;
 		prev_ndx = F_HL_PREV(file);
 		prev_name = f_name(file, NULL);

io.c

@@ -53,6 +53,9 @@ extern int preserve_hard_links;
 extern char *filesfrom_host;
 extern struct stats stats;
 extern struct file_list *cur_flist, *first_flist;
+#ifdef ICONV_OPTION
+extern iconv_t ic_send, ic_recv;
+#endif
 
 const char phase_unknown[] = "unknown";
 int ignore_timeout = 0;
@@ -91,6 +94,9 @@ static int write_batch_monitor_out = -1;
 static int io_filesfrom_f_in = -1;
 static int io_filesfrom_f_out = -1;
 static char io_filesfrom_buf[2048];
+#ifdef ICONV_OPTION
+static char iconv_buf[sizeof io_filesfrom_buf / 2];
+#endif
 static char *io_filesfrom_bp;
 static char io_filesfrom_lastchar;
 static int io_filesfrom_buflen;
@@ -111,6 +117,7 @@ static void writefd(int fd, const char *buf, size_t len);
 static void writefd_unbuffered(int fd, const char *buf, size_t len);
 static void decrement_active_files(int ndx);
 static void decrement_flist_in_progress(int ndx, int redo);
+static void mplex_write(int fd, enum msgcode code, const char *buf, size_t len, int convert);
 
 struct flist_ndx_item {
 	struct flist_ndx_item *next;
@@ -125,7 +132,7 @@ static struct flist_ndx_list redo_list, hlink_list;
 
 struct msg_list_item {
 	struct msg_list_item *next;
-	int len;
+	char convert;
 	char buf[1];
 };
 
@@ -229,7 +236,7 @@ void set_msg_fd_out(int fd)
 }
 
 /* Add a message to the pending MSG_* list. */
-static void msg_list_add(struct msg_list *lst, int code, const char *buf, int len)
+static void msg_list_add(struct msg_list *lst, int code, const char *buf, int len, int convert)
 {
 	struct msg_list_item *m;
 	int sz = len + 4 + sizeof m[0] - 1;
@@ -237,7 +244,7 @@ static void msg_list_add(struct msg_list *lst, int code, const char *buf, int le
 	if (!(m = (struct msg_list_item *)new_array(char, sz)))
 		out_of_memory("msg_list_add");
 	m->next = NULL;
-	m->len = len + 4;
+	m->convert = convert;
 	SIVAL(m->buf, 0, ((code+MPLEX_BASE)<<24) | len);
 	memcpy(m->buf + 4, buf, len);
 	if (lst->tail)
@@ -252,21 +259,25 @@ static void msg_flush(void)
 	if (am_generator) {
 		while (msg_queue.head && io_multiplexing_out) {
 			struct msg_list_item *m = msg_queue.head;
+			int len = IVAL(m->buf, 0) & 0xFFFFFF;
+			int tag = *((uchar*)m->buf+3) - MPLEX_BASE;
 			if (!(msg_queue.head = m->next))
 				msg_queue.tail = NULL;
-			stats.total_written += m->len;
+			stats.total_written += len + 4;
 			defer_forwarding_messages++;
-			writefd_unbuffered(sock_f_out, m->buf, m->len);
+			mplex_write(sock_f_out, tag, m->buf + 4, len, m->convert);
 			defer_forwarding_messages--;
 			free(m);
 		}
 	} else {
 		while (msg_queue.head) {
 			struct msg_list_item *m = msg_queue.head;
+			int len = IVAL(m->buf, 0) & 0xFFFFFF;
+			int tag = *((uchar*)m->buf+3) - MPLEX_BASE;
 			if (!(msg_queue.head = m->next))
 				msg_queue.tail = NULL;
 			defer_forwarding_messages++;
-			writefd_unbuffered(msg_fd_out, m->buf, m->len);
+			mplex_write(msg_fd_out, tag, m->buf + 4, len, m->convert);
 			defer_forwarding_messages--;
 			free(m);
 		}
@@ -341,7 +352,7 @@ static void read_msg_fd(void)
 		if (len >= (int)sizeof buf || !am_generator)
 			goto invalid_msg;
 		readfd(fd, buf, len);
-		send_msg(MSG_DELETED, buf, len);
+		send_msg(MSG_DELETED, buf, len, 1);
 		break;
 	case MSG_SUCCESS:
 		if (len != 4 || !am_generator)
@@ -349,7 +360,7 @@ static void read_msg_fd(void)
 		readfd(fd, buf, len);
 		if (remove_source_files) {
 			decrement_active_files(IVAL(buf,0));
-			send_msg(MSG_SUCCESS, buf, len);
+			send_msg(MSG_SUCCESS, buf, len, 0);
 		}
 		if (preserve_hard_links)
 			flist_ndx_push(&hlink_list, IVAL(buf,0));
@@ -378,7 +389,7 @@ static void read_msg_fd(void)
 			if (n >= sizeof buf)
 				n = sizeof buf - 1;
 			readfd(fd, buf, n);
-			rwrite((enum logcode)tag, buf, n);
+			rwrite((enum logcode)tag, buf, n, !am_generator);
 			len -= n;
 		}
 		break;
@@ -447,14 +458,19 @@ static void decrement_flist_in_progress(int ndx, int redo)
 }
 
 /* Write an message to a multiplexed stream. If this fails, rsync exits. */
-static void mplex_write(int fd, enum msgcode code, const char *buf, size_t len)
+static void mplex_write(int fd, enum msgcode code, const char *buf, size_t len, int convert)
 {
-	char buffer[1024];
+	char buffer[BIGPATHBUFLEN]; /* Oversized for use by iconv code. */
 	size_t n = len;
 
 	SIVAL(buffer, 0, ((MPLEX_BASE + (int)code)<<24) + len);
 
-	if (n > sizeof buffer - 4)
+#ifdef ICONV_OPTION
+	if (convert && ic_send == (iconv_t)-1)
+#endif
+		convert = 0;
+
+	if (convert || n > 1024 - 4) /* BIGPATHBUFLEN can handle 1024 bytes */
 		n = 0;
 	else
 		memcpy(buffer + 4, buf, n);
@@ -464,6 +480,28 @@ static void mplex_write(int fd, enum msgcode code, const char *buf, size_t len)
 	len -= n;
 	buf += n;
 
+#ifdef ICONV_OPTION
+	if (convert) {
+		iconv(ic_send, NULL, 0, NULL, 0);
+		defer_forwarding_messages++;
+		while (len) {
+			ICONV_CONST char *ibuf = (ICONV_CONST char *)buf;
+			char *obuf = buffer;
+			size_t ocnt = sizeof buffer;
+			while (len && iconv(ic_send, &ibuf,&len,
+					    &obuf,&ocnt) == (size_t)-1) {
+				if (errno == E2BIG || !ocnt)
+					break;
+				*obuf++ = *ibuf++;
+				ocnt--, len--;
+			}
+			n = obuf - buffer;
+			writefd_unbuffered(fd, buffer, n);
+		}
+		if (!--defer_forwarding_messages)
+			msg_flush();
+	} else
+#endif
 	if (len) {
 		defer_forwarding_messages++;
 		writefd_unbuffered(fd, buf, len);
@@ -472,20 +510,20 @@ static void mplex_write(int fd, enum msgcode code, const char *buf, size_t len)
 	}
 }
 
-int send_msg(enum msgcode code, const char *buf, int len)
+int send_msg(enum msgcode code, const char *buf, int len, int convert)
 {
 	if (msg_fd_out < 0) {
 		if (!defer_forwarding_messages)
-			return io_multiplex_write(code, buf, len);
+			return io_multiplex_write(code, buf, len, convert);
 		if (!io_multiplexing_out)
 			return 0;
-		msg_list_add(&msg_queue, code, buf, len);
+		msg_list_add(&msg_queue, code, buf, len, convert);
 		return 1;
 	}
 	if (flist_forward_from >= 0)
-		msg_list_add(&msg_queue, code, buf, len);
+		msg_list_add(&msg_queue, code, buf, len, convert);
 	else
-		mplex_write(msg_fd_out, code, buf, len);
+		mplex_write(msg_fd_out, code, buf, len, convert);
 	return 1;
 }
 
@@ -493,7 +531,7 @@ void send_msg_int(enum msgcode code, int num)
 {
 	char numbuf[4];
 	SIVAL(numbuf, 0, num);
-	send_msg(code, numbuf, 4);
+	send_msg(code, numbuf, 4, 0);
 }
 
 void wait_for_receiver(void)
@@ -719,10 +757,8 @@ static int read_timeout(int fd, char *buf, size_t len)
 	return cnt;
 }
 
-/**
+/* Read a line into the "fname" buffer (which must be at least MAXPATHLEN
- * Read a line into the "fname" buffer (which must be at least MAXPATHLEN
+ * characters long). */
- * characters long).
- */
 int read_filesfrom_line(int fd, char *fname)
 {
 	char ch, *s, *eob = fname + MAXPATHLEN - 1;
@@ -833,7 +869,7 @@ void maybe_send_keepalive(void)
 			if (protocol_version < 29)
 				return; /* there's nothing we can do */
 			if (protocol_version >= 30)
-				send_msg(MSG_NOOP, "", 0);
+				send_msg(MSG_NOOP, "", 0, 0);
 			else {
 				write_int(sock_f_out, cur_flist->count);
 				write_shortint(sock_f_out, ITEM_IS_NEW);
@@ -932,7 +968,30 @@ static int readfd_unbuffered(int fd, char *buf, size_t len)
 		case MSG_DELETED:
 			if (msg_bytes >= sizeof line)
 				goto overflow;
-			read_loop(fd, line, msg_bytes);
+#ifdef ICONV_OPTION
+			if (ic_recv != (iconv_t)-1) {
+				ICONV_CONST char *ibuf;
+				char *obuf = line;
+				size_t icnt, ocnt = sizeof line - 1;
+				int add_null = 0;
+				iconv(ic_send, NULL, 0, NULL, 0);
+				while (msg_bytes) {
+					icnt = msg_bytes > sizeof iconv_buf
+					     ? sizeof iconv_buf : msg_bytes;
+					read_loop(fd, iconv_buf, icnt);
+					if (!(msg_bytes -= icnt) && !iconv_buf[icnt-1])
+						icnt--, add_null = 1;
+					ibuf = (ICONV_CONST char *)iconv_buf;
+					if (iconv(ic_send, &ibuf,&icnt,
+						  &obuf,&ocnt) == (size_t)-1)
+						goto overflow; // XXX
+				}
+				if (add_null)
+					*obuf++ = '\0';
+				msg_bytes = obuf - line;
+			} else
+#endif
+				read_loop(fd, line, msg_bytes);
 			/* A directory name was sent with the trailing null */
 			if (msg_bytes > 0 && !line[msg_bytes-1])
 				log_delete(line, S_IFDIR);
@@ -967,7 +1026,7 @@ static int readfd_unbuffered(int fd, char *buf, size_t len)
 				exit_cleanup(RERR_STREAMIO);
 			}
 			read_loop(fd, line, msg_bytes);
-			rwrite((enum logcode)tag, line, msg_bytes);
+			rwrite((enum logcode)tag, line, msg_bytes, 1);
 			break;
 		default:
 			rprintf(FERROR, "unexpected tag %d [%s]\n",
@@ -1372,7 +1431,7 @@ void io_flush(int flush_it_all)
 		return;
 
 	if (io_multiplexing_out)
-		mplex_write(sock_f_out, MSG_DATA, iobuf_out, iobuf_out_cnt);
+		mplex_write(sock_f_out, MSG_DATA, iobuf_out, iobuf_out_cnt, 0);
 	else
 		writefd_unbuffered(iobuf_f_out, iobuf_out, iobuf_out_cnt);
 	iobuf_out_cnt = 0;
@@ -1684,13 +1743,13 @@ void io_start_multiplex_in(void)
 }
 
 /** Write an message to the multiplexed data stream. */
-int io_multiplex_write(enum msgcode code, const char *buf, size_t len)
+int io_multiplex_write(enum msgcode code, const char *buf, size_t len, int convert)
 {
 	if (!io_multiplexing_out)
 		return 0;
 	io_flush(NORMAL_FLUSH);
 	stats.total_written += (len+4);
-	mplex_write(sock_f_out, code, buf, len);
+	mplex_write(sock_f_out, code, buf, len, convert);
 	return 1;
 }
 

log.c

@@ -20,15 +20,13 @@
  */
 
 #include "rsync.h"
-#if defined HAVE_ICONV_OPEN && defined HAVE_ICONV_H
-#include <iconv.h>
-#endif
 
 extern int verbose;
 extern int dry_run;
 extern int am_daemon;
 extern int am_server;
 extern int am_sender;
+extern int am_generator;
 extern int local_server;
 extern int quiet;
 extern int module_id;
@@ -47,9 +45,12 @@ extern char *auth_user;
 extern char *stdout_format;
 extern char *logfile_format;
 extern char *logfile_name;
-#if defined HAVE_ICONV_OPEN && defined HAVE_ICONV_H
+#ifdef ICONV_CONST
 extern iconv_t ic_chck;
 #endif
+#ifdef ICONV_OPTION
+extern iconv_t ic_send, ic_recv;
+#endif
 extern char curr_dir[];
 extern unsigned int module_dirlen;
 
@@ -234,17 +235,25 @@ static void filtered_fwrite(FILE *f, const char *buf, int len, int use_isprint)
 /* this is the underlying (unformatted) rsync debugging function. Call
  * it with FINFO, FERROR or FLOG.  Note: recursion can happen with
  * certain fatal conditions. */
-void rwrite(enum logcode code, const char *buf, int len)
+void rwrite(enum logcode code, const char *buf, int len, int is_utf8)
 {
 	int trailing_CR_or_NL;
 	FILE *f = NULL;
+#ifdef ICONV_OPTION
+	iconv_t ic = is_utf8 && ic_recv != (iconv_t)-1 ? ic_recv : ic_chck;
+#else
+#ifdef ICONV_CONST
+	iconv_t ic = ic_chck;
+#endif
+#endif
 
 	if (len < 0)
 		exit_cleanup(RERR_MESSAGEIO);
 
 	if (am_server && msg_fd_out >= 0) {
+		assert(!is_utf8);
 		/* Pass the message to our sibling. */
-		send_msg((enum msgcode)code, buf, len);
+		send_msg((enum msgcode)code, buf, len, 0);
 		return;
 	}
 
@@ -277,7 +286,7 @@ void rwrite(enum logcode code, const char *buf, int len)
 
 	if (am_server) {
 		/* Pass the message to the non-server side. */
-		if (send_msg((enum msgcode)code, buf, len))
+		if (send_msg((enum msgcode)code, buf, len, !is_utf8))
 			return;
 		if (am_daemon) {
 			/* TODO: can we send the error to the user somehow? */
@@ -300,18 +309,15 @@ void rwrite(enum logcode code, const char *buf, int len)
 	trailing_CR_or_NL = len && (buf[len-1] == '\n' || buf[len-1] == '\r')
 			  ? buf[--len] : 0;
 
-#if defined HAVE_ICONV_OPEN && defined HAVE_ICONV_H
+#ifdef ICONV_CONST
-#ifndef ICONV_CONST
+	if (ic != (iconv_t)-1) {
-#define ICONV_CONST
-#endif
-	if (ic_chck != (iconv_t)-1) {
 		char convbuf[1024];
 		ICONV_CONST char *in_buf = (ICONV_CONST char *)buf;
 		char *out_buf = convbuf;
 		size_t in_cnt = len, out_cnt = sizeof convbuf - 1;
 
-		iconv(ic_chck, NULL, 0, NULL, 0);
+		iconv(ic, NULL, 0, NULL, 0);
-		while (iconv(ic_chck, &in_buf,&in_cnt,
+		while (iconv(ic, &in_buf,&in_cnt,
 			     &out_buf,&out_cnt) == (size_t)-1) {
 			if (out_buf != convbuf) {
 				filtered_fwrite(f, convbuf, out_buf - convbuf, 0);
@@ -373,7 +379,7 @@ void rprintf(enum logcode code, const char *format, ...)
 		}
 	}
 
-	rwrite(code, buf, len);
+	rwrite(code, buf, len, 0);
 }
 
 /* This is like rprintf, but it also tries to print some
@@ -404,7 +410,7 @@ void rsyserr(enum logcode code, int errcode, const char *format, ...)
 	if (len >= sizeof buf)
 		exit_cleanup(RERR_MESSAGEIO);
 
-	rwrite(code, buf, len);
+	rwrite(code, buf, len, 0);
 }
 
 void rflush(enum logcode code)
@@ -681,7 +687,7 @@ static void log_formatted(enum logcode code, const char *format, const char *op,
 		p = s + len;
 	}
 
-	rwrite(code, buf, total);
+	rwrite(code, buf, total, 0);
 }
 
 /* Return 1 if the format escape is in the log-format string (e.g. look for
@@ -758,7 +764,7 @@ void log_delete(const char *fname, int mode)
 	else if (am_server && protocol_version >= 29 && len < MAXPATHLEN) {
 		if (S_ISDIR(mode))
 			len++; /* directories include trailing null */
-		send_msg(MSG_DELETED, fname, len);
+		send_msg(MSG_DELETED, fname, len, am_generator);
 	} else {
 		fmt = stdout_format_has_o_or_i ? stdout_format : "deleting %n";
 		log_formatted(FCLIENT, fmt, "del.", &x.file, fname, &stats,

main.c

@@ -727,7 +727,7 @@ static int do_recv(int f_in, int f_out, char *local_name)
 		io_flush(FULL_FLUSH);
 		handle_stats(f_in);
 
-		send_msg(MSG_DONE, "", 1);
+		send_msg(MSG_DONE, "", 1, 0);
 		write_varlong(error_pipe[1], stats.total_read, 3);
 		io_flush(FULL_FLUSH);
 
@@ -900,7 +900,7 @@ void start_server(int f_in, int f_out, int argc, char *argv[])
 
 	io_set_sock_fds(f_in, f_out);
 	setup_protocol(f_out, f_in);
-#if defined HAVE_ICONV_OPEN && defined HAVE_ICONV_H
+#ifdef ICONV_CONST
 	setup_iconv();
 #endif
 
@@ -937,7 +937,7 @@ int client_run(int f_in, int f_out, pid_t pid, int argc, char *argv[])
 
 	io_set_sock_fds(f_in, f_out);
 	setup_protocol(f_out,f_in);
-#if defined HAVE_ICONV_OPEN && defined HAVE_ICONV_H
+#ifdef ICONV_CONST
 	setup_iconv();
 #endif
 

options.c

@@ -177,6 +177,11 @@ int list_only = 0;
 #define MAX_BATCH_NAME_LEN 256	/* Must be less than MAXPATHLEN-13 */
 char *batch_name = NULL;
 
+#ifdef ICONV_OPTION
+int need_unsorted_flist = 0;
+char *iconv_opt = ICONV_OPTION;
+#endif
+
 struct chmod_mode_struct *chmod_modes = NULL;
 
 static int daemon_opt;   /* sets am_daemon after option error-reporting */
@@ -205,6 +210,7 @@ static void print_rsync_version(enum logcode f)
 	char const *acls = "no ";
 	char const *xattrs = "no ";
 	char const *links = "no ";
+	char const *iconv = "no ";
 	char const *ipv6 = "no ";
 	STRUCT_STAT *dumstat;
 
@@ -232,6 +238,9 @@ static void print_rsync_version(enum logcode f)
 #ifdef INET6
 	ipv6 = "";
 #endif
+#ifdef ICONV_OPTION
+	iconv = "";
+#endif
 
 	rprintf(f, "%s  version %s  protocol version %d%s\n",
 		RSYNC_NAME, RSYNC_VERSION, PROTOCOL_VERSION, subprotocol);
@@ -245,8 +254,8 @@ static void print_rsync_version(enum logcode f)
 		(int)(sizeof (int64) * 8));
 	rprintf(f, "    %ssocketpairs, %shardlinks, %ssymlinks, %sIPv6, batchfiles, %sinplace,\n",
 		got_socketpair, hardlinks, links, ipv6, have_inplace);
-	rprintf(f, "    %sappend, %sACLs, %sxattrs\n",
+	rprintf(f, "    %sappend, %sACLs, %sxattrs, %siconv\n",
-		have_inplace, acls, xattrs);
+		have_inplace, acls, xattrs, iconv);
 
 #ifdef MAINTAINER_MODE
 	rprintf(f, "Panic Action: \"%s\"\n", get_panic_action());
@@ -399,6 +408,9 @@ void usage(enum logcode F)
   rprintf(F,"     --only-write-batch=FILE like --write-batch but w/o updating destination\n");
   rprintf(F,"     --read-batch=FILE       read a batched update from FILE\n");
   rprintf(F,"     --protocol=NUM          force an older protocol version to be used\n");
+#ifdef ICONV_OPTION
+  rprintf(F,"     --iconv=CONVERT_SPEC    request charset conversion of filesnames\n");
+#endif
 #ifdef INET6
   rprintf(F," -4, --ipv4                  prefer IPv4\n");
   rprintf(F," -6, --ipv6                  prefer IPv6\n");
@@ -560,6 +572,9 @@ static struct poptOption long_options[] = {
   {"rsh",             'e', POPT_ARG_STRING, &shell_cmd, 0, 0, 0 },
   {"rsync-path",       0,  POPT_ARG_STRING, &rsync_path, 0, 0, 0 },
   {"temp-dir",        'T', POPT_ARG_STRING, &tmpdir, 0, 0, 0 },
+#ifdef ICONV_OPTION
+  {"iconv",            0,  POPT_ARG_STRING, &iconv_opt, 0, 0, 0 },
+#endif
 #ifdef INET6
   {"ipv4",            '4', POPT_ARG_VAL,    &default_af_hint, AF_INET, 0, 0 },
   {"ipv6",            '6', POPT_ARG_VAL,    &default_af_hint, AF_INET6, 0, 0 },
@@ -832,6 +847,11 @@ int parse_arguments(int *argc, const char ***argv, int frommain)
 	if (am_daemon)
 		set_refuse_options("log-file*");
 
+#ifdef ICONV_OPTION
+	if (!am_daemon && (arg = getenv("RSYNC_ICONV")) != NULL && *arg)
+		iconv_opt = strdup(arg);
+#endif
+
 	/* TODO: Call poptReadDefaultConfig; handle errors. */
 
 	/* The context leaks in case of an error, but if there's a
@@ -857,6 +877,9 @@ int parse_arguments(int *argc, const char ***argv, int frommain)
 						    long_options, 0);
 				am_server = 1;
 			}
+#ifdef ICONV_OPTION
+			iconv_opt = NULL;
+#endif
 			break;
 
 		case OPT_SENDER:
@@ -874,6 +897,9 @@ int parse_arguments(int *argc, const char ***argv, int frommain)
 					sizeof err_buf);
 				return 0;
 			}
+#ifdef ICONV_OPTION
+			iconv_opt = NULL;
+#endif
 			poptFreeContext(pc);
 			pc = poptGetContext(RSYNC_NAME, *argc, *argv,
 					    long_daemon_options, 0);
@@ -1175,6 +1201,15 @@ int parse_arguments(int *argc, const char ***argv, int frommain)
 		exit_cleanup(0);
 	}
 
+#ifdef ICONV_OPTION
+	if (iconv_opt) {
+		if (!am_server && strcmp(iconv_opt, "-") == 0)
+			iconv_opt = NULL;
+		else
+			need_unsorted_flist = 1;
+	}
+#endif
+
 #ifndef SUPPORT_LINKS
 	if (preserve_links && !am_sender) {
 		snprintf(err_buf, sizeof err_buf,
@@ -1655,8 +1690,6 @@ void server_options(char **args,int *argc)
 	if (list_only == 1 && !recurse)
 		argstr[x++] = 'r';
 
-	argstr[x] = '\0';
-
 #if SUBPROTOCOL_VERSION != 0
 	/* If we're speaking a pre-release version of a protocol, we tell
 	 * the server about this by (ab)using the -e option. */
@@ -1666,6 +1699,8 @@ void server_options(char **args,int *argc)
 	}
 #endif
 
+	argstr[x] = '\0';
+
 	if (x != 1)
 		args[ac++] = argstr;
 
@@ -1704,6 +1739,19 @@ void server_options(char **args,int *argc)
 			args[ac++] = "--log-format=X";
 	}
 
+#ifdef ICONV_OPTION
+	if (iconv_opt) {
+		char *set = strchr(iconv_opt, ',');
+		if (set)
+			set++;
+		else
+			set = iconv_opt;
+		if (asprintf(&arg, "--iconv=%s", set) < 0)
+			goto oom;
+		args[ac++] = arg;
+	}
+#endif
+
 	if (block_size) {
 		if (asprintf(&arg, "-B%lu", block_size) < 0)
 			goto oom;

receiver.c

@@ -389,7 +389,7 @@ int recv_files(int f_in, char *local_name)
 				rprintf(FINFO, "recv_files phase=%d\n", phase);
 			if (phase == 2 && delay_updates)
 				handle_delayed_updates(local_name);
-			send_msg(MSG_DONE, "", 0);
+			send_msg(MSG_DONE, "", 0, 0);
 			continue;
 		}
 

rsync.c

@@ -20,9 +20,6 @@
  */
 
 #include "rsync.h"
-#if defined HAVE_ICONV_OPEN && defined HAVE_ICONV_H
-#include <iconv.h>
-#endif
 #if defined HAVE_LIBCHARSET_H && defined HAVE_LOCALE_CHARSET
 #include <libcharset.h>
 #elif defined HAVE_LANGINFO_H && defined HAVE_NL_LANGINFO
@@ -53,9 +50,16 @@ extern int keep_dirlinks;
 extern int make_backups;
 extern struct file_list *cur_flist, *first_flist, *dir_flist;
 extern struct chmod_mode_struct *daemon_chmod_modes;
+#ifdef ICONV_OPTION
+extern char *iconv_opt;
+#endif
 
-#if defined HAVE_ICONV_OPEN && defined HAVE_ICONV_H
+#ifdef ICONV_CONST
 iconv_t ic_chck = (iconv_t)-1;
+#ifdef ICONV_OPTION
+iconv_t ic_send = (iconv_t)-1, ic_recv = (iconv_t)-1;
+int ic_ndx;
+#endif
 
 static const char *default_charset(void)
 {
@@ -70,8 +74,13 @@ static const char *default_charset(void)
 
 void setup_iconv()
 {
+	const char *defset = default_charset();
+# ifdef ICONV_OPTION
+	const char *charset;
+	char *cp;
+#endif
+
 	if (!am_server && !allow_8bit_chars) {
-		const char *defset = default_charset();
 
 		/* It's OK if this fails... */
 		ic_chck = iconv_open(defset, defset);
@@ -89,6 +98,44 @@ void setup_iconv()
 			}
 		}
 	}
+
+# ifdef ICONV_OPTION
+	if (!iconv_opt)
+		return;
+
+	if ((cp = strchr(iconv_opt, ',')) != NULL) {
+		if (am_server) /* A local transfer needs this. */
+			iconv_opt = cp + 1;
+		else
+			*cp = '\0';
+	}
+
+	if (!*iconv_opt || (*iconv_opt == '.' && iconv_opt[1] == '\0'))
+		charset = defset;
+	else
+		charset = iconv_opt;
+
+	if ((ic_send = iconv_open(UTF8_CHARSET, charset)) == (iconv_t)-1) {
+		rprintf(FERROR, "iconv_open(\"%s\", \"%s\") failed\n",
+			UTF8_CHARSET, charset);
+		exit_cleanup(RERR_UNSUPPORTED);
+	}
+
+	if ((ic_recv = iconv_open(charset, UTF8_CHARSET)) == (iconv_t)-1) {
+		rprintf(FERROR, "iconv_open(\"%s\", \"%s\") failed\n",
+			charset, UTF8_CHARSET);
+		exit_cleanup(RERR_UNSUPPORTED);
+	}
+
+	if (!am_sender)
+		ic_ndx = ++file_extra_cnt;
+
+	if (verbose > 1) {
+		rprintf(FINFO, "%s charset: %s\n",
+			am_server ? "server" : "client",
+			*charset ? charset : "[LOCALE]");
+	}
+# endif
 }
 #endif
 
@@ -112,7 +159,7 @@ int read_ndx_and_attrs(int f_in, int *iflag_ptr, uchar *type_ptr,
 			goto invalid_ndx;
 		if (ndx == NDX_FLIST_EOF) {
 			flist_eof = 1;
-			send_msg(MSG_FLIST_EOF, "", 0);
+			send_msg(MSG_FLIST_EOF, "", 0, 0);
 			continue;
 		}
 		ndx = NDX_FLIST_OFFSET - ndx;

rsync.h

@@ -362,6 +362,15 @@ enum msgcode {
 # include <limits.h>
 #endif
 
+#if defined HAVE_ICONV_OPEN && defined HAVE_ICONV_H
+#include <iconv.h>
+#ifndef ICONV_CONST
+#define ICONV_CONST
+#endif
+#elif defined ICONV_CONST
+#undef ICONV_CONST
+#endif
+
 #include <assert.h>
 
 #include "lib/pool_alloc.h"
@@ -609,6 +618,7 @@ extern int preserve_xattrs;
 #define F_GROUP(f) REQ_EXTRA(f, preserve_gid)->unum
 #define F_ACL(f) REQ_EXTRA(f, preserve_acls)->num
 #define F_XATTR(f) REQ_EXTRA(f, preserve_xattrs)->num
+#define F_NDX(f) REQ_EXTRA(f, ic_ndx)->num
 
 /* These items are per-entry optional and mutally exclusive: */
 #define F_HL_GNUM(f) OPT_EXTRA(f, LEN64_BUMP(f))->num
@@ -665,7 +675,7 @@ extern int preserve_xattrs;
 
 struct file_list {
 	struct file_list *next, *prev;
-	struct file_struct **files;
+	struct file_struct **files, **sorted;
 	alloc_pool_t file_pool;
 	int count, malloced;
 	int low, high; /* 0-relative index values excluding empties */

rsync.yo

@@ -402,6 +402,7 @@ to the detailed description below for a complete description.  verb(
      --only-write-batch=FILE like --write-batch but w/o updating dest
      --read-batch=FILE       read a batched update from FILE
      --protocol=NUM          force an older protocol version to be used
+     --iconv=CONVERT_SPEC    request charset conversion of filesnames
      --checksum-seed=NUM     set block/file checksum seed (advanced)
  -4, --ipv4                  prefer IPv4
  -6, --ipv6                  prefer IPv6
@@ -1866,6 +1867,24 @@ bf(--read-batch) option, you should use "--protocol=28" when creating the
 batch file to force the older protocol version to be used in the batch
 file (assuming you can't upgrade the rsync on the reading system).
 
+dit(bf(--iconv=CONVERT_SPEC)) Rsync can convert filenames between character
+sets using this option.  Using a CONVERT_SPEC of "." tells rsync to look up
+the default character-set via the locale setting.  Alternately, you can
+fully specify what conversion to do by giving a local and a remote charset
+separated by a comma (local first), e.g. bf(--iconv=utf8,iso88591).
+Finally, you can specify a CONVERT_SPEC of "-" to turn off any conversion.
+The default setting of this option is site-specific, and can also be
+affected via the RSYNC_ICONV environment variable.
+
+Note that rsync does not do any conversion of names in filter files
+(including include/exclude files), in a files-from file, nor those
+specified on the command line.  It is up to you to ensure that you're
+requesting the right names from a remote server, and you can specify
+extra include/exclude rules if there are filename differences on the
+two sides that need to be accounted for.  (In the future there may be
+a way to specify a UTF-8 filter rule that gets auto-converted to the
+local side's character set.)
+
 dit(bf(-4, --ipv4) or bf(-6, --ipv6)) Tells rsync to prefer IPv4/IPv6
 when creating sockets.  This only affects sockets that rsync has direct
 control over, such as the outgoing socket when directly contacting an
@@ -2617,6 +2636,8 @@ startdit()
 dit(bf(CVSIGNORE)) The CVSIGNORE environment variable supplements any
 ignore patterns in .cvsignore files. See the bf(--cvs-exclude) option for
 more details.
+dit(bf(RSYNC_ICONV)) Specify a default bf(--iconv) setting using this
+environment variable.
 dit(bf(RSYNC_RSH)) The RSYNC_RSH environment variable allows you to
 override the default shell used as the transport for rsync.  Command line
 options are permitted after the command name, just as in the bf(-e) option.

socket.c

@@ -409,7 +409,7 @@ static int *open_socket_in(int type, int port, const char *bind_addr,
 	 * unsuccessful, or if the daemon is being run with -vv. */
 	for (s = 0; s < ecnt; s++) {
 		if (!i || verbose > 1)
-			rwrite(FLOG, errmsgs[s], strlen(errmsgs[s]));
+			rwrite(FLOG, errmsgs[s], strlen(errmsgs[s]), 0);
 		free(errmsgs[s]);
 	}
 	free(errmsgs);

testsuite/rsync.fns

@@ -56,6 +56,8 @@ filter_outfile() {
 	-e '/^done$/d' \
 	-e '/ --whole-file$/d' \
 	-e '/^total: /d' \
+	-e '/^client charset: /d' \
+	-e '/^server charset: /d' \
 	-e '/^$/,$d' \
 	<"$outfile" >"$outfile.new"
     mv "$outfile.new" "$outfile"