mirrors/mc
1
0
Fork 0
mirror of https://github.com/MidnightCommander/mc synced 2025-01-10 21:42:00 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch '2193_copy_buffer_size'

Browse Source 
* 2193_copy_buffer_size:
  (copy_file_file): reduce variable scopes.
  (copy_file_file): get rid of one goto.
  (copy_file_file): rename variables.
  (copy_file_file): define copy buffer size in depend on destination file system block size.
  (io_blksize): add intermediate variable.
  (ST_BLKSIZE): fix signed and unsigned comparision.
  Ticket #2193: use Coreutils way to define buffer size to copy file.
This commit is contained in:
Andrew Borodin 2016-04-25 13:29:26 +03:00
commit 7b928e6b94
8 changed files with 257 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
acinclude.m4
View File

@ -4,6 +4,7 @@ m4_include([m4.include/dx_doxygen.m4])
m4_include([m4.include/mc-cflags.m4])
m4_include([m4.include/mc-check-search-type.m4])
m4_include([m4.include/mode_t.m4])
m4_include([m4.include/stat-size.m4])
m4_include([m4.include/ls-mntd-fs.m4])
m4_include([m4.include/fstypename.m4])
m4_include([m4.include/fsusage.m4])

1
configure.ac
View File

@ -183,6 +183,7 @@ AC_TYPE_UID_T
AC_STRUCT_ST_BLOCKS
AC_CHECK_MEMBERS([struct stat.st_blksize, struct stat.st_rdev])
gl_STAT_SIZE
AH_TEMPLATE([sig_atomic_t],
[/* Define to `int' if <signal.h> doesn't define.])

1
lib/Makefile.am
View File

@ -41,6 +41,7 @@ libmc_la_SOURCES = \
lock.c lock.h \
serialize.c serialize.h \
shell.c shell.h \
stat-size.h \
timefmt.c timefmt.h \
timer.c timer.h

111
lib/stat-size.h Normal file
View File

@ -0,0 +1,111 @@
/* macros useful in interpreting size-related values in struct stat.
Copyright (C) 1989, 1991-2016 Free Software Foundation, Inc.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>. */
/* *INDENT-OFF* */
/*
Macros defined by this file (s is an rvalue of type struct stat):
DEV_BSIZE: The device blocksize. But use ST_NBLOCKSIZE instead.
ST_BLKSIZE(s): Preferred (in the sense of best performance) I/O blocksize
for the file, in bytes.
ST_NBLOCKS(s): Number of blocks in the file, including indirect blocks.
ST_NBLOCKSIZE: Size of blocks used when calculating ST_NBLOCKS.
*/
/* *INDENT-ON* */
#ifndef STAT_SIZE_H
#define STAT_SIZE_H
/* sys/param.h may define DEV_BSIZE */
#if HAVE_SYS_PARAM_H
#include <sys/param.h>
#endif
/* Get or fake the disk device blocksize.
Usually defined by sys/param.h (if at all). */
#if !defined DEV_BSIZE && defined BSIZE
#define DEV_BSIZE BSIZE
#endif
#if !defined DEV_BSIZE && defined BBSIZE /* SGI sys/param.h */
#define DEV_BSIZE BBSIZE
#endif
#ifndef DEV_BSIZE
#define DEV_BSIZE 4096
#endif
/* Extract or fake data from a 'struct stat'.
ST_BLKSIZE: Preferred I/O blocksize for the file, in bytes.
ST_NBLOCKS: Number of blocks in the file, including indirect blocks.
ST_NBLOCKSIZE: Size of blocks used when calculating ST_NBLOCKS. */
#ifndef HAVE_STRUCT_STAT_ST_BLOCKS
#define ST_BLKSIZE(statbuf) DEV_BSIZE
/* coreutils' fileblocks.c also uses BSIZE. */
#if defined _POSIX_SOURCE || !defined BSIZE
#define ST_NBLOCKS(statbuf) \
((statbuf).st_size / ST_NBLOCKSIZE + ((statbuf).st_size % ST_NBLOCKSIZE != 0))
#else
/* This definition calls st_blocks, which is in the fileblocks module. */
#define ST_NBLOCKS(statbuf) \
(S_ISREG ((statbuf).st_mode) || S_ISDIR ((statbuf).st_mode) ? \
st_blocks ((statbuf).st_size) : 0)
#endif
#else
/* Some systems, like Sequents, return st_blksize of 0 on pipes.
Also, when running 'rsh hpux11-system cat any-file', cat would
determine that the output stream had an st_blksize of 2147421096.
Conversely st_blksize can be 2 GiB (or maybe even larger) with XFS
on 64-bit hosts. Somewhat arbitrarily, limit the "optimal" block
size to SIZE_MAX / 8 + 1. (Dividing SIZE_MAX by only 4 wouldn't
suffice, since "cat" sometimes multiplies the result by 4.) If
anyone knows of a system for which this limit is too small, please
report it as a bug in this code. */
#define ST_BLKSIZE(statbuf) ((0 < (statbuf).st_blksize \
&& (size_t) ((statbuf).st_blksize) <= ((size_t)-1) / 8 + 1) \
? (size_t) ((statbuf).st_blksize) : DEV_BSIZE)
#if defined hpux || defined __hpux__ || defined __hpux
/* HP-UX counts st_blocks in 1024-byte units.
This loses when mixing HP-UX and BSD file systems with NFS. */
#define ST_NBLOCKSIZE 1024
#else /* !hpux */
#if defined _AIX && defined _I386
/* AIX PS/2 counts st_blocks in 4K units. */
#define ST_NBLOCKSIZE (4 * 1024)
#else
#if defined _CRAY
#define ST_NBLOCKS(statbuf) \
(S_ISREG ((statbuf).st_mode) || S_ISDIR ((statbuf).st_mode) \
? (statbuf).st_blocks * ST_BLKSIZE (statbuf) / ST_NBLOCKSIZE : 0)
#endif
#endif
#endif
#endif
#ifndef ST_NBLOCKS
#define ST_NBLOCKS(statbuf) ((statbuf).st_blocks)
#endif
#ifndef ST_NBLOCKSIZE
#ifdef S_BLKSIZE
#define ST_NBLOCKSIZE S_BLKSIZE
#else
#define ST_NBLOCKSIZE 512
#endif
#endif
#endif /* STAT_SIZE_H */

14
m4.include/stat-size.m4 Normal file
View File

@ -0,0 +1,14 @@
#serial 1
# Copyright (C) 2011-2016 Free Software Foundation, Inc.
#
# This file is free software; the Free Software Foundation
# gives unlimited permission to copy and/or distribute it,
# with or without modifications, as long as this notice is preserved.
AC_DEFUN([gl_STAT_SIZE],
[
# Don't call AC_STRUCT_ST_BLOCKS because it causes bugs. Details at
# http://lists.gnu.org/archive/html/bug-gnulib/2011-06/msg00051.html
AC_CHECK_HEADERS_ONCE([sys/param.h])
])

1
src/filemanager/Makefile.am
View File

@ -17,6 +17,7 @@ libmcfilemanager_la_SOURCES = \
find.c find.h \
hotlist.c hotlist.h \
info.c info.h \
ioblksize.h \
layout.c layout.h \
midnight.h midnight.c \
mountlist.c mountlist.h \

78
src/filemanager/file.c
View File

@ -82,6 +82,7 @@
#include "tree.h"
#include "midnight.h" /* current_panel */
#include "layout.h" /* rotate_dash() */
#include "ioblksize.h" /* io_blksize() */
#include "file.h"
@ -1485,9 +1486,8 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx,
gid_t src_gid = (gid_t) (-1);
int src_desc, dest_desc = -1;
int n_read, n_written;
mode_t src_mode = 0; /* The mode of the source file */
struct stat sb, sb2;
struct stat src_stat, dst_stat;
struct utimbuf utb;
gboolean dst_exists = FALSE, appending = FALSE;
off_t file_size = -1;
@ -1495,8 +1495,8 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx,
struct timeval tv_transfer_start;
dest_status_t dst_status = DEST_NONE;
int open_flags;
gboolean is_first_time = TRUE;
vfs_path_t *src_vpath = NULL, *dst_vpath = NULL;
char *buf = NULL;
/* FIXME: We should not be using global variables! */
ctx->do_reget = 0;
@ -1516,9 +1516,9 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx,
mc_refresh ();
while (mc_stat (dst_vpath, &sb2) == 0)
while (mc_stat (dst_vpath, &dst_stat) == 0)
{
if (S_ISDIR (sb2.st_mode))
if (S_ISDIR (dst_stat.st_mode))
{
if (ctx->skip_all)
return_status = FILE_SKIPALL;
@ -1537,7 +1537,7 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx,
break;
}
while ((*ctx->stat_func) (src_vpath, &sb) != 0)
while ((*ctx->stat_func) (src_vpath, &src_stat) != 0)
{
if (ctx->skip_all)
return_status = FILE_SKIPALL;
@ -1555,7 +1555,7 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx,
if (dst_exists)
{
/* Destination already exists */
if (sb.st_dev == sb2.st_dev && sb.st_ino == sb2.st_ino)
if (src_stat.st_dev == dst_stat.st_dev && src_stat.st_ino == dst_stat.st_ino)
{
return_status = warn_same_file (_("\"%s\"\nand\n\"%s\"\nare the same file"),
src_path, dst_path);
@ -1566,7 +1566,7 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx,
if (tctx->ask_overwrite)
{
ctx->do_reget = 0;
return_status = query_replace (ctx, dst_path, &sb, &sb2);
return_status = query_replace (ctx, dst_path, &src_stat, &dst_stat);
if (return_status != FILE_CONT)
goto ret_fast;
}
@ -1575,23 +1575,24 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx,
if (!ctx->do_append)
{
/* Check the hardlinks */
if (!ctx->follow_links && sb.st_nlink > 1 && check_hardlinks (src_vpath, dst_vpath, &sb))
if (!ctx->follow_links && src_stat.st_nlink > 1
&& check_hardlinks (src_vpath, dst_vpath, &src_stat))
{
/* We have made a hardlink - no more processing is necessary */
return_status = FILE_CONT;
goto ret_fast;
}
if (S_ISLNK (sb.st_mode))
if (S_ISLNK (src_stat.st_mode))
{
return_status = make_symlink (ctx, src_path, dst_path);
goto ret_fast;
}
if (S_ISCHR (sb.st_mode) || S_ISBLK (sb.st_mode) ||
S_ISFIFO (sb.st_mode) || S_ISNAM (sb.st_mode) || S_ISSOCK (sb.st_mode))
if (S_ISCHR (src_stat.st_mode) || S_ISBLK (src_stat.st_mode) || S_ISFIFO (src_stat.st_mode)
|| S_ISNAM (src_stat.st_mode) || S_ISSOCK (src_stat.st_mode))
{
while (mc_mknod (dst_vpath, sb.st_mode & ctx->umask_kill, sb.st_rdev) < 0
while (mc_mknod (dst_vpath, src_stat.st_mode & ctx->umask_kill, src_stat.st_rdev) < 0
&& !ctx->skip_all)
{
return_status = file_error (_("Cannot create special file \"%s\"\n%s"), dst_path);
@ -1603,8 +1604,8 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx,
}
/* Success */
while (ctx->preserve_uidgid && mc_chown (dst_vpath, sb.st_uid, sb.st_gid) != 0
&& !ctx->skip_all)
while (ctx->preserve_uidgid
&& mc_chown (dst_vpath, src_stat.st_uid, src_stat.st_gid) != 0 && !ctx->skip_all)
{
temp_status = file_error (_("Cannot chown target file \"%s\"\n%s"), dst_path);
if (temp_status == FILE_SKIP)
@ -1618,7 +1619,7 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx,
}
}
while (ctx->preserve && mc_chmod (dst_vpath, sb.st_mode & ctx->umask_kill) != 0
while (ctx->preserve && mc_chmod (dst_vpath, src_stat.st_mode & ctx->umask_kill) != 0
&& !ctx->skip_all)
{
temp_status = file_error (_("Cannot chmod target file \"%s\"\n%s"), dst_path);
@ -1663,7 +1664,7 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx,
}
}
while (mc_fstat (src_desc, &sb) != 0)
while (mc_fstat (src_desc, &src_stat) != 0)
{
if (ctx->skip_all)
return_status = FILE_SKIPALL;
@ -1679,12 +1680,12 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx,
goto ret;
}
src_mode = sb.st_mode;
src_uid = sb.st_uid;
src_gid = sb.st_gid;
utb.actime = sb.st_atime;
utb.modtime = sb.st_mtime;
file_size = sb.st_size;
src_mode = src_stat.st_mode;
src_uid = src_stat.st_uid;
src_gid = src_stat.st_gid;
utb.actime = src_stat.st_atime;
utb.modtime = src_stat.st_mtime;
file_size = src_stat.st_size;
open_flags = O_WRONLY;
if (dst_exists)
@ -1723,7 +1724,7 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx,
ctx->do_append = FALSE;
/* Find out the optimal buffer size. */
while (mc_fstat (dest_desc, &sb) != 0)
while (mc_fstat (dest_desc, &dst_stat) != 0)
{
if (ctx->skip_all)
return_status = FILE_SKIPALL;
@ -1739,7 +1740,7 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx,
}
/* try preallocate space; if fail, try copy anyway */
while (vfs_preallocate (dest_desc, file_size, appending ? sb.st_size : 0) != 0)
while (vfs_preallocate (dest_desc, file_size, appending ? dst_stat.st_size : 0) != 0)
{
if (ctx->skip_all)
{
@ -1783,26 +1784,27 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx,
return_status = check_progress_buttons (ctx);
mc_refresh ();
if (return_status != FILE_CONT)
goto ret;
if (return_status == FILE_CONT)
{
size_t bufsize;
off_t n_read_total = 0;
struct timeval tv_current, tv_last_update, tv_last_input;
int secs, update_secs;
const char *stalled_msg = "";
gboolean is_first_time = TRUE;
tv_last_update = tv_transfer_start;
bufsize = io_blksize (dst_stat);
buf = g_malloc (bufsize);
while (TRUE)
{
char buf[BUF_8K];
ssize_t n_read = -1, n_written;
/* src_read */
if (mc_ctl (src_desc, VFS_CTL_IS_NOTREADY, 0))
n_read = -1;
else
while ((n_read = mc_read (src_desc, buf, sizeof (buf))) < 0 && !ctx->skip_all)
if (mc_ctl (src_desc, VFS_CTL_IS_NOTREADY, 0) == 0)
while ((n_read = mc_read (src_desc, buf, bufsize)) < 0 && !ctx->skip_all)
{
return_status = file_error (_("Cannot read source file \"%s\"\n%s"), src_path);
if (return_status == FILE_RETRY)
@ -1811,6 +1813,7 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx,
ctx->skip_all = TRUE;
goto ret;
}
if (n_read == 0)
break;
@ -1819,6 +1822,7 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx,
if (n_read > 0)
{
char *t = buf;
n_read_total += n_read;
/* Windows NT ftp servers report that files have no
@ -1830,7 +1834,7 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx,
gettimeofday (&tv_last_input, NULL);
/* dst_write */
while ((n_written = mc_write (dest_desc, t, n_read)) < n_read)
while ((n_written = mc_write (dest_desc, t, (size_t) n_read)) < n_read)
{
gboolean write_errno_nospace;
@ -1910,11 +1914,13 @@ copy_file_file (file_op_total_context_t * tctx, file_op_context_t * ctx,
goto ret;
}
}
dst_status = DEST_FULL; /* copy successful, don't remove target file */
}
dst_status = DEST_FULL; /* copy successful, don't remove target file */
ret:
g_free (buf);
rotate_dash (FALSE);
while (src_desc != -1 && mc_close (src_desc) < 0 && !ctx->skip_all)
{

86
src/filemanager/ioblksize.h Normal file
View File

@ -0,0 +1,86 @@
/* I/O block size definitions for coreutils
Copyright (C) 1989-2016 Free Software Foundation, Inc.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>. */
/* Include this file _after_ system headers if possible. */
/* sys/stat.h will already have been included by system.h. */
#include "lib/stat-size.h"
/* *INDENT-OFF* */
/* As of May 2014, 128KiB is determined to be the minimium
blksize to best minimize system call overhead.
This can be tested with this script:
for i in $(seq 0 10); do
bs=$((1024*2**$i))
printf "%7s=" $bs
timeout --foreground -sINT 2 \
dd bs=$bs if=/dev/zero of=/dev/null 2>&1 \
| sed -n 's/.* \([0-9.]* [GM]B\/s\)/\1/p'
done
With the results shown for these systems:
system #1: 1.7GHz pentium-m with 400MHz DDR2 RAM, arch=i686
system #2: 2.1GHz i3-2310M with 1333MHz DDR3 RAM, arch=x86_64
system #3: 3.2GHz i7-970 with 1333MHz DDR3, arch=x86_64
system #4: 2.20GHz Xeon E5-2660 with 1333MHz DDR3, arch=x86_64
system #5: 2.30GHz i7-3615QM with 1600MHz DDR3, arch=x86_64
system #6: 1.30GHz i5-4250U with 1-channel 1600MHz DDR3, arch=x86_64
system #7: 3.55GHz IBM,8231-E2B with 1066MHz DDR3, POWER7 revision 2.1
per-system transfer rate (GB/s)
blksize #1 #2 #3 #4 #5 #6 #7
------------------------------------------------------------------------
1024 .73 1.7 2.6 .64 1.0 2.5 1.3
2048 1.3 3.0 4.4 1.2 2.0 4.4 2.5
4096 2.4 5.1 6.5 2.3 3.7 7.4 4.8
8192 3.5 7.3 8.5 4.0 6.0 10.4 9.2
16384 3.9 9.4 10.1 6.3 8.3 13.3 16.8
32768 5.2 9.9 11.1 8.1 10.7 13.2 28.0
65536 5.3 11.2 12.0 10.6 12.8 16.1 41.4
131072 5.5 11.8 12.3 12.1 14.0 16.7 54.8
262144 5.7 11.6 12.5 12.3 14.7 16.4 40.0
524288 5.7 11.4 12.5 12.1 14.7 15.5 34.5
1048576 5.8 11.4 12.6 12.2 14.9 15.7 36.5
Note that this is to minimize system call overhead.
Other values may be appropriate to minimize file system
or disk overhead. For example on my current GNU/Linux system
the readahead setting is 128KiB which was read using:
file="."
device=$(df --output=source --local "$file" | tail -n1)
echo $(( $(blockdev --getra $device) * 512 ))
However there isn't a portable way to get the above.
In the future we could use the above method if available
and default to io_blksize() if not.
*/
enum { IO_BUFSIZE = 128 * 1024 };
/* *INDENT-ON* */
static inline size_t
io_blksize (struct stat sb)
{
size_t blksize = ST_BLKSIZE (sb);
return MAX (IO_BUFSIZE, blksize);
}