diff --git a/acinclude.m4 b/acinclude.m4 index 861ca2261..fcdf6b28a 100644 --- a/acinclude.m4 +++ b/acinclude.m4 @@ -4,6 +4,7 @@ m4_include([m4.include/dx_doxygen.m4]) m4_include([m4.include/mc-cflags.m4]) m4_include([m4.include/mc-check-search-type.m4]) m4_include([m4.include/mode_t.m4]) +m4_include([m4.include/stat-size.m4]) m4_include([m4.include/ls-mntd-fs.m4]) m4_include([m4.include/fstypename.m4]) m4_include([m4.include/fsusage.m4]) diff --git a/configure.ac b/configure.ac index d0d8358cc..3c7ca181f 100644 --- a/configure.ac +++ b/configure.ac @@ -183,6 +183,7 @@ AC_TYPE_UID_T AC_STRUCT_ST_BLOCKS AC_CHECK_MEMBERS([struct stat.st_blksize, struct stat.st_rdev]) +gl_STAT_SIZE AH_TEMPLATE([sig_atomic_t], [/* Define to `int' if doesn't define.]) diff --git a/lib/Makefile.am b/lib/Makefile.am index 8a419280b..0c08e7ddd 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -41,6 +41,7 @@ libmc_la_SOURCES = \ lock.c lock.h \ serialize.c serialize.h \ shell.c shell.h \ + stat-size.h \ timefmt.c timefmt.h \ timer.c timer.h diff --git a/lib/stat-size.h b/lib/stat-size.h new file mode 100644 index 000000000..682c1b2d4 --- /dev/null +++ b/lib/stat-size.h @@ -0,0 +1,111 @@ +/* macros useful in interpreting size-related values in struct stat. + Copyright (C) 1989, 1991-2016 Free Software Foundation, Inc. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . */ + +/* *INDENT-OFF* */ +/* + Macros defined by this file (s is an rvalue of type struct stat): + + DEV_BSIZE: The device blocksize. But use ST_NBLOCKSIZE instead. + ST_BLKSIZE(s): Preferred (in the sense of best performance) I/O blocksize + for the file, in bytes. + ST_NBLOCKS(s): Number of blocks in the file, including indirect blocks. + ST_NBLOCKSIZE: Size of blocks used when calculating ST_NBLOCKS. + */ +/* *INDENT-ON* */ + +#ifndef STAT_SIZE_H +#define STAT_SIZE_H + +/* sys/param.h may define DEV_BSIZE */ +#if HAVE_SYS_PARAM_H +#include +#endif + + +/* Get or fake the disk device blocksize. + Usually defined by sys/param.h (if at all). */ +#if !defined DEV_BSIZE && defined BSIZE +#define DEV_BSIZE BSIZE +#endif +#if !defined DEV_BSIZE && defined BBSIZE /* SGI sys/param.h */ +#define DEV_BSIZE BBSIZE +#endif +#ifndef DEV_BSIZE +#define DEV_BSIZE 4096 +#endif + + + +/* Extract or fake data from a 'struct stat'. + ST_BLKSIZE: Preferred I/O blocksize for the file, in bytes. + ST_NBLOCKS: Number of blocks in the file, including indirect blocks. + ST_NBLOCKSIZE: Size of blocks used when calculating ST_NBLOCKS. */ +#ifndef HAVE_STRUCT_STAT_ST_BLOCKS +#define ST_BLKSIZE(statbuf) DEV_BSIZE + /* coreutils' fileblocks.c also uses BSIZE. */ +#if defined _POSIX_SOURCE || !defined BSIZE +#define ST_NBLOCKS(statbuf) \ + ((statbuf).st_size / ST_NBLOCKSIZE + ((statbuf).st_size % ST_NBLOCKSIZE != 0)) +#else + /* This definition calls st_blocks, which is in the fileblocks module. */ +#define ST_NBLOCKS(statbuf) \ + (S_ISREG ((statbuf).st_mode) || S_ISDIR ((statbuf).st_mode) ? \ + st_blocks ((statbuf).st_size) : 0) +#endif +#else +/* Some systems, like Sequents, return st_blksize of 0 on pipes. + Also, when running 'rsh hpux11-system cat any-file', cat would + determine that the output stream had an st_blksize of 2147421096. + Conversely st_blksize can be 2 GiB (or maybe even larger) with XFS + on 64-bit hosts. Somewhat arbitrarily, limit the "optimal" block + size to SIZE_MAX / 8 + 1. (Dividing SIZE_MAX by only 4 wouldn't + suffice, since "cat" sometimes multiplies the result by 4.) If + anyone knows of a system for which this limit is too small, please + report it as a bug in this code. */ +#define ST_BLKSIZE(statbuf) ((0 < (statbuf).st_blksize \ + && (size_t) ((statbuf).st_blksize) <= ((size_t)-1) / 8 + 1) \ + ? (size_t) ((statbuf).st_blksize) : DEV_BSIZE) +#if defined hpux || defined __hpux__ || defined __hpux + /* HP-UX counts st_blocks in 1024-byte units. + This loses when mixing HP-UX and BSD file systems with NFS. */ +#define ST_NBLOCKSIZE 1024 +#else /* !hpux */ +#if defined _AIX && defined _I386 + /* AIX PS/2 counts st_blocks in 4K units. */ +#define ST_NBLOCKSIZE (4 * 1024) +#else +#if defined _CRAY +#define ST_NBLOCKS(statbuf) \ + (S_ISREG ((statbuf).st_mode) || S_ISDIR ((statbuf).st_mode) \ + ? (statbuf).st_blocks * ST_BLKSIZE (statbuf) / ST_NBLOCKSIZE : 0) +#endif +#endif +#endif +#endif + +#ifndef ST_NBLOCKS +#define ST_NBLOCKS(statbuf) ((statbuf).st_blocks) +#endif + +#ifndef ST_NBLOCKSIZE +#ifdef S_BLKSIZE +#define ST_NBLOCKSIZE S_BLKSIZE +#else +#define ST_NBLOCKSIZE 512 +#endif +#endif + +#endif /* STAT_SIZE_H */ diff --git a/m4.include/stat-size.m4 b/m4.include/stat-size.m4 new file mode 100644 index 000000000..86d409864 --- /dev/null +++ b/m4.include/stat-size.m4 @@ -0,0 +1,14 @@ +#serial 1 + +# Copyright (C) 2011-2016 Free Software Foundation, Inc. +# +# This file is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +AC_DEFUN([gl_STAT_SIZE], +[ + # Don't call AC_STRUCT_ST_BLOCKS because it causes bugs. Details at + # http://lists.gnu.org/archive/html/bug-gnulib/2011-06/msg00051.html + AC_CHECK_HEADERS_ONCE([sys/param.h]) +]) diff --git a/src/filemanager/Makefile.am b/src/filemanager/Makefile.am index 7188dc500..b593b7025 100644 --- a/src/filemanager/Makefile.am +++ b/src/filemanager/Makefile.am @@ -17,6 +17,7 @@ libmcfilemanager_la_SOURCES = \ find.c find.h \ hotlist.c hotlist.h \ info.c info.h \ + ioblksize.h \ layout.c layout.h \ midnight.h midnight.c \ mountlist.c mountlist.h \ diff --git a/src/filemanager/file.c b/src/filemanager/file.c index fd3fef6c7..2460b0f9f 100644 --- a/src/filemanager/file.c +++ b/src/filemanager/file.c @@ -82,6 +82,7 @@ #include "tree.h" #include "midnight.h" /* current_panel */ #include "layout.h" /* rotate_dash() */ +#include "ioblksize.h" /* io_blksize() */ #include "file.h" @@ -1485,9 +1486,8 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx, gid_t src_gid = (gid_t) (-1); int src_desc, dest_desc = -1; - int n_read, n_written; mode_t src_mode = 0; /* The mode of the source file */ - struct stat sb, sb2; + struct stat src_stat, dst_stat; struct utimbuf utb; gboolean dst_exists = FALSE, appending = FALSE; off_t file_size = -1; @@ -1495,8 +1495,8 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx, struct timeval tv_transfer_start; dest_status_t dst_status = DEST_NONE; int open_flags; - gboolean is_first_time = TRUE; vfs_path_t *src_vpath = NULL, *dst_vpath = NULL; + char *buf = NULL; /* FIXME: We should not be using global variables! */ ctx->do_reget = 0; @@ -1516,9 +1516,9 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx, mc_refresh (); - while (mc_stat (dst_vpath, &sb2) == 0) + while (mc_stat (dst_vpath, &dst_stat) == 0) { - if (S_ISDIR (sb2.st_mode)) + if (S_ISDIR (dst_stat.st_mode)) { if (ctx->skip_all) return_status = FILE_SKIPALL; @@ -1537,7 +1537,7 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx, break; } - while ((*ctx->stat_func) (src_vpath, &sb) != 0) + while ((*ctx->stat_func) (src_vpath, &src_stat) != 0) { if (ctx->skip_all) return_status = FILE_SKIPALL; @@ -1555,7 +1555,7 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx, if (dst_exists) { /* Destination already exists */ - if (sb.st_dev == sb2.st_dev && sb.st_ino == sb2.st_ino) + if (src_stat.st_dev == dst_stat.st_dev && src_stat.st_ino == dst_stat.st_ino) { return_status = warn_same_file (_("\"%s\"\nand\n\"%s\"\nare the same file"), src_path, dst_path); @@ -1566,7 +1566,7 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx, if (tctx->ask_overwrite) { ctx->do_reget = 0; - return_status = query_replace (ctx, dst_path, &sb, &sb2); + return_status = query_replace (ctx, dst_path, &src_stat, &dst_stat); if (return_status != FILE_CONT) goto ret_fast; } @@ -1575,23 +1575,24 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx, if (!ctx->do_append) { /* Check the hardlinks */ - if (!ctx->follow_links && sb.st_nlink > 1 && check_hardlinks (src_vpath, dst_vpath, &sb)) + if (!ctx->follow_links && src_stat.st_nlink > 1 + && check_hardlinks (src_vpath, dst_vpath, &src_stat)) { /* We have made a hardlink - no more processing is necessary */ return_status = FILE_CONT; goto ret_fast; } - if (S_ISLNK (sb.st_mode)) + if (S_ISLNK (src_stat.st_mode)) { return_status = make_symlink (ctx, src_path, dst_path); goto ret_fast; } - if (S_ISCHR (sb.st_mode) || S_ISBLK (sb.st_mode) || - S_ISFIFO (sb.st_mode) || S_ISNAM (sb.st_mode) || S_ISSOCK (sb.st_mode)) + if (S_ISCHR (src_stat.st_mode) || S_ISBLK (src_stat.st_mode) || S_ISFIFO (src_stat.st_mode) + || S_ISNAM (src_stat.st_mode) || S_ISSOCK (src_stat.st_mode)) { - while (mc_mknod (dst_vpath, sb.st_mode & ctx->umask_kill, sb.st_rdev) < 0 + while (mc_mknod (dst_vpath, src_stat.st_mode & ctx->umask_kill, src_stat.st_rdev) < 0 && !ctx->skip_all) { return_status = file_error (_("Cannot create special file \"%s\"\n%s"), dst_path); @@ -1603,8 +1604,8 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx, } /* Success */ - while (ctx->preserve_uidgid && mc_chown (dst_vpath, sb.st_uid, sb.st_gid) != 0 - && !ctx->skip_all) + while (ctx->preserve_uidgid + && mc_chown (dst_vpath, src_stat.st_uid, src_stat.st_gid) != 0 && !ctx->skip_all) { temp_status = file_error (_("Cannot chown target file \"%s\"\n%s"), dst_path); if (temp_status == FILE_SKIP) @@ -1618,7 +1619,7 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx, } } - while (ctx->preserve && mc_chmod (dst_vpath, sb.st_mode & ctx->umask_kill) != 0 + while (ctx->preserve && mc_chmod (dst_vpath, src_stat.st_mode & ctx->umask_kill) != 0 && !ctx->skip_all) { temp_status = file_error (_("Cannot chmod target file \"%s\"\n%s"), dst_path); @@ -1663,7 +1664,7 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx, } } - while (mc_fstat (src_desc, &sb) != 0) + while (mc_fstat (src_desc, &src_stat) != 0) { if (ctx->skip_all) return_status = FILE_SKIPALL; @@ -1679,12 +1680,12 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx, goto ret; } - src_mode = sb.st_mode; - src_uid = sb.st_uid; - src_gid = sb.st_gid; - utb.actime = sb.st_atime; - utb.modtime = sb.st_mtime; - file_size = sb.st_size; + src_mode = src_stat.st_mode; + src_uid = src_stat.st_uid; + src_gid = src_stat.st_gid; + utb.actime = src_stat.st_atime; + utb.modtime = src_stat.st_mtime; + file_size = src_stat.st_size; open_flags = O_WRONLY; if (dst_exists) @@ -1723,7 +1724,7 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx, ctx->do_append = FALSE; /* Find out the optimal buffer size. */ - while (mc_fstat (dest_desc, &sb) != 0) + while (mc_fstat (dest_desc, &dst_stat) != 0) { if (ctx->skip_all) return_status = FILE_SKIPALL; @@ -1739,7 +1740,7 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx, } /* try preallocate space; if fail, try copy anyway */ - while (vfs_preallocate (dest_desc, file_size, appending ? sb.st_size : 0) != 0) + while (vfs_preallocate (dest_desc, file_size, appending ? dst_stat.st_size : 0) != 0) { if (ctx->skip_all) { @@ -1783,26 +1784,27 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx, return_status = check_progress_buttons (ctx); mc_refresh (); - if (return_status != FILE_CONT) - goto ret; - + if (return_status == FILE_CONT) { + size_t bufsize; off_t n_read_total = 0; struct timeval tv_current, tv_last_update, tv_last_input; int secs, update_secs; const char *stalled_msg = ""; + gboolean is_first_time = TRUE; tv_last_update = tv_transfer_start; + bufsize = io_blksize (dst_stat); + buf = g_malloc (bufsize); + while (TRUE) { - char buf[BUF_8K]; + ssize_t n_read = -1, n_written; /* src_read */ - if (mc_ctl (src_desc, VFS_CTL_IS_NOTREADY, 0)) - n_read = -1; - else - while ((n_read = mc_read (src_desc, buf, sizeof (buf))) < 0 && !ctx->skip_all) + if (mc_ctl (src_desc, VFS_CTL_IS_NOTREADY, 0) == 0) + while ((n_read = mc_read (src_desc, buf, bufsize)) < 0 && !ctx->skip_all) { return_status = file_error (_("Cannot read source file \"%s\"\n%s"), src_path); if (return_status == FILE_RETRY) @@ -1811,6 +1813,7 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx, ctx->skip_all = TRUE; goto ret; } + if (n_read == 0) break; @@ -1819,6 +1822,7 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx, if (n_read > 0) { char *t = buf; + n_read_total += n_read; /* Windows NT ftp servers report that files have no @@ -1830,7 +1834,7 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx, gettimeofday (&tv_last_input, NULL); /* dst_write */ - while ((n_written = mc_write (dest_desc, t, n_read)) < n_read) + while ((n_written = mc_write (dest_desc, t, (size_t) n_read)) < n_read) { gboolean write_errno_nospace; @@ -1910,11 +1914,13 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx, goto ret; } } + + dst_status = DEST_FULL; /* copy successful, don't remove target file */ } - dst_status = DEST_FULL; /* copy successful, don't remove target file */ - ret: + g_free (buf); + rotate_dash (FALSE); while (src_desc != -1 && mc_close (src_desc) < 0 && !ctx->skip_all) { diff --git a/src/filemanager/ioblksize.h b/src/filemanager/ioblksize.h new file mode 100644 index 000000000..3a8b50181 --- /dev/null +++ b/src/filemanager/ioblksize.h @@ -0,0 +1,86 @@ +/* I/O block size definitions for coreutils + Copyright (C) 1989-2016 Free Software Foundation, Inc. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . */ + +/* Include this file _after_ system headers if possible. */ + +/* sys/stat.h will already have been included by system.h. */ +#include "lib/stat-size.h" + +/* *INDENT-OFF* */ + +/* As of May 2014, 128KiB is determined to be the minimium + blksize to best minimize system call overhead. + This can be tested with this script: + + for i in $(seq 0 10); do + bs=$((1024*2**$i)) + printf "%7s=" $bs + timeout --foreground -sINT 2 \ + dd bs=$bs if=/dev/zero of=/dev/null 2>&1 \ + | sed -n 's/.* \([0-9.]* [GM]B\/s\)/\1/p' + done + + With the results shown for these systems: + system #1: 1.7GHz pentium-m with 400MHz DDR2 RAM, arch=i686 + system #2: 2.1GHz i3-2310M with 1333MHz DDR3 RAM, arch=x86_64 + system #3: 3.2GHz i7-970 with 1333MHz DDR3, arch=x86_64 + system #4: 2.20GHz Xeon E5-2660 with 1333MHz DDR3, arch=x86_64 + system #5: 2.30GHz i7-3615QM with 1600MHz DDR3, arch=x86_64 + system #6: 1.30GHz i5-4250U with 1-channel 1600MHz DDR3, arch=x86_64 + system #7: 3.55GHz IBM,8231-E2B with 1066MHz DDR3, POWER7 revision 2.1 + + per-system transfer rate (GB/s) + blksize #1 #2 #3 #4 #5 #6 #7 + ------------------------------------------------------------------------ + 1024 .73 1.7 2.6 .64 1.0 2.5 1.3 + 2048 1.3 3.0 4.4 1.2 2.0 4.4 2.5 + 4096 2.4 5.1 6.5 2.3 3.7 7.4 4.8 + 8192 3.5 7.3 8.5 4.0 6.0 10.4 9.2 + 16384 3.9 9.4 10.1 6.3 8.3 13.3 16.8 + 32768 5.2 9.9 11.1 8.1 10.7 13.2 28.0 + 65536 5.3 11.2 12.0 10.6 12.8 16.1 41.4 + 131072 5.5 11.8 12.3 12.1 14.0 16.7 54.8 + 262144 5.7 11.6 12.5 12.3 14.7 16.4 40.0 + 524288 5.7 11.4 12.5 12.1 14.7 15.5 34.5 + 1048576 5.8 11.4 12.6 12.2 14.9 15.7 36.5 + + + Note that this is to minimize system call overhead. + Other values may be appropriate to minimize file system + or disk overhead. For example on my current GNU/Linux system + the readahead setting is 128KiB which was read using: + + file="." + device=$(df --output=source --local "$file" | tail -n1) + echo $(( $(blockdev --getra $device) * 512 )) + + However there isn't a portable way to get the above. + In the future we could use the above method if available + and default to io_blksize() if not. + */ + + +enum { IO_BUFSIZE = 128 * 1024 }; + +/* *INDENT-ON* */ + +static inline size_t +io_blksize (struct stat sb) +{ + size_t blksize = ST_BLKSIZE (sb); + + return MAX (IO_BUFSIZE, blksize); +}