From a18a7fa39662dca1cbb07c70f34b1f874d241c27 Mon Sep 17 00:00:00 2001 From: Andrew Borodin Date: Tue, 2 Apr 2019 14:14:10 +0300 Subject: [PATCH] Ticket #77: fish: get rid of copy file to tmp before transfer [travis ci skip]. NOT FINISHED! Signed-off-by: Andrew Borodin --- src/vfs/fish/fish.c | 390 +++++++++++++-------------------------- src/vfs/fish/fishdef.h | 24 ++- src/vfs/fish/helpers/get | 87 +++++---- 3 files changed, 193 insertions(+), 308 deletions(-) diff --git a/src/vfs/fish/fish.c b/src/vfs/fish/fish.c index f73f0e3f6..b9efbc564 100644 --- a/src/vfs/fish/fish.c +++ b/src/vfs/fish/fish.c @@ -151,7 +151,7 @@ typedef struct { vfs_file_handler_t base; /* base class */ - off_t got; + char *file_name; off_t total; gboolean append; } fish_file_handler_t; @@ -962,233 +962,6 @@ fish_dir_load (struct vfs_class *me, struct vfs_s_inode *dir, char *remote_path) /* --------------------------------------------------------------------------------------------- */ -static int -fish_file_store (struct vfs_class *me, vfs_file_handler_t * fh, char *name, char *localname) -{ - fish_file_handler_t *fish = FISH_FILE_HANDLER (fh); - struct vfs_s_super *super = VFS_FILE_HANDLER_SUPER (fh); - fish_super_t *fish_super = FISH_SUPER (super); - int code; - off_t total = 0; - char buffer[BUF_8K]; - struct stat s; - int h; - char *quoted_name; - - h = open (localname, O_RDONLY); - if (h == -1) - ERRNOR (EIO, -1); - if (fstat (h, &s) < 0) - { - close (h); - ERRNOR (EIO, -1); - } - - /* First, try this as stor: - * - * ( head -c number ) | ( cat > file; cat >/dev/null ) - * - * If 'head' is not present on the remote system, 'dd' will be used. - * Unfortunately, we cannot trust most non-GNU 'head' implementations - * even if '-c' options is supported. Therefore, we separate GNU head - * (and other modern heads?) using '-q' and '-' . This causes another - * implementations to fail (because of "incorrect options"). - * - * Fallback is: - * - * rest= - * while [ $rest -gt 0 ] - * do - * cnt=`expr \( $rest + 255 \) / 256` - * n=`dd bs=256 count=$cnt | tee -a | wc -c` - * rest=`expr $rest - $n` - * done - * - * 'dd' was not designed for full filling of input buffers, - * and does not report exact number of bytes (not blocks). - * Therefore a more complex shell script is needed. - * - * On some systems non-GNU head writes "Usage:" error report to stdout - * instead of stderr. It makes impossible the use of "head || dd" - * algorithm for file appending case, therefore just "dd" is used for it. - */ - - quoted_name = strutils_shell_escape (name); - vfs_print_message (_("fish: store %s: sending command..."), quoted_name); - - /* FIXME: File size is limited to ULONG_MAX */ - code = - fish_command_v (me, super, WAIT_REPLY, - fish->append ? fish_super->scr_append : fish_super->scr_send, - "FISH_FILENAME=%s FISH_FILESIZE=%" PRIuMAX ";\n", quoted_name, - (uintmax_t) s.st_size); - g_free (quoted_name); - - if (code != PRELIM) - { - close (h); - ERRNOR (E_REMOTE, -1); - } - - while (TRUE) - { - ssize_t n, t; - - while ((n = read (h, buffer, sizeof (buffer))) < 0) - { - if ((errno == EINTR) && tty_got_interrupt ()) - continue; - vfs_print_message ("%s", _("fish: Local read failed, sending zeros")); - close (h); - h = open ("/dev/zero", O_RDONLY); - } - - if (n == 0) - break; - - t = write (fish_super->sockw, buffer, n); - if (t != n) - { - if (t == -1) - me->verrno = errno; - else - me->verrno = EIO; - goto error_return; - } - tty_disable_interrupt_key (); - total += n; - vfs_print_message ("%s: %" PRIuMAX "/%" PRIuMAX, _("fish: storing file"), - (uintmax_t) total, (uintmax_t) s.st_size); - } - close (h); - - if (fish_get_reply (me, fish_super->sockr, NULL, 0) != COMPLETE) - ERRNOR (E_REMOTE, -1); - return 0; - - error_return: - close (h); - fish_get_reply (me, fish_super->sockr, NULL, 0); - return -1; -} - -/* --------------------------------------------------------------------------------------------- */ - -static int -fish_linear_start (struct vfs_class *me, vfs_file_handler_t * fh, off_t offset) -{ - fish_file_handler_t *fish = FISH_FILE_HANDLER (fh); - struct vfs_s_super *super = VFS_FILE_HANDLER_SUPER (fh); - char *name; - char *quoted_name; - - name = vfs_s_fullpath (me, fh->ino); - if (name == NULL) - return 0; - quoted_name = strutils_shell_escape (name); - g_free (name); - fish->append = FALSE; - - /* - * Check whether the remote file is readable by using 'dd' to copy - * a single byte from the remote file to /dev/null. If 'dd' completes - * with exit status of 0 use 'cat' to send the file contents to the - * standard output (i.e. over the network). - */ - - offset = - fish_command_v (me, super, WANT_STRING, FISH_SUPER (super)->scr_get, - "FISH_FILENAME=%s FISH_START_OFFSET=%" PRIuMAX ";\n", quoted_name, - (uintmax_t) offset); - g_free (quoted_name); - - if (offset != PRELIM) - ERRNOR (E_REMOTE, 0); - fh->linear = LS_LINEAR_OPEN; - fish->got = 0; - errno = 0; -#if SIZEOF_OFF_T == SIZEOF_LONG - fish->total = (off_t) strtol (reply_str, NULL, 10); -#else - fish->total = (off_t) g_ascii_strtoll (reply_str, NULL, 10); -#endif - if (errno != 0) - ERRNOR (E_REMOTE, 0); - return 1; -} - -/* --------------------------------------------------------------------------------------------- */ - -static void -fish_linear_abort (struct vfs_class *me, vfs_file_handler_t * fh) -{ - fish_file_handler_t *fish = FISH_FILE_HANDLER (fh); - struct vfs_s_super *super = VFS_FILE_HANDLER_SUPER (fh); - char buffer[BUF_8K]; - ssize_t n; - - vfs_print_message ("%s", _("Aborting transfer...")); - - do - { - n = MIN ((off_t) sizeof (buffer), (fish->total - fish->got)); - if (n != 0) - { - n = read (FISH_SUPER (super)->sockr, buffer, n); - if (n < 0) - return; - fish->got += n; - } - } - while (n != 0); - - if (fish_get_reply (me, FISH_SUPER (super)->sockr, NULL, 0) != COMPLETE) - vfs_print_message ("%s", _("Error reported after abort.")); - else - vfs_print_message ("%s", _("Aborted transfer would be successful.")); -} - -/* --------------------------------------------------------------------------------------------- */ - -static ssize_t -fish_linear_read (struct vfs_class *me, vfs_file_handler_t * fh, void *buf, size_t len) -{ - fish_file_handler_t *fish = FISH_FILE_HANDLER (fh); - struct vfs_s_super *super = VFS_FILE_HANDLER_SUPER (fh); - ssize_t n = 0; - - len = MIN ((size_t) (fish->total - fish->got), len); - tty_disable_interrupt_key (); - while (len != 0 && ((n = read (FISH_SUPER (super)->sockr, buf, len)) < 0)) - { - if ((errno == EINTR) && !tty_got_interrupt ()) - continue; - break; - } - tty_enable_interrupt_key (); - - if (n > 0) - fish->got += n; - else if (n < 0) - fish_linear_abort (me, fh); - else if (fish_get_reply (me, FISH_SUPER (super)->sockr, NULL, 0) != COMPLETE) - ERRNOR (E_REMOTE, -1); - ERRNOR (errno, n); -} - -/* --------------------------------------------------------------------------------------------- */ - -static void -fish_linear_close (struct vfs_class *me, vfs_file_handler_t * fh) -{ - fish_file_handler_t *fish = FISH_FILE_HANDLER (fh); - - if (fish->total != fish->got) - fish_linear_abort (me, fh); -} - -/* --------------------------------------------------------------------------------------------- */ - static int fish_ctl (void *fh, int ctlop, void *arg) { @@ -1203,14 +976,8 @@ fish_ctl (void *fh, int ctlop, void *arg) { case VFS_CTL_IS_NOTREADY: { - vfs_file_handler_t *file = VFS_FILE_HANDLER (fh); int v; - if (file->linear == LS_NOT_LINEAR) - vfs_die ("You may not do this"); - if (file->linear == LS_LINEAR_CLOSED || file->linear == LS_LINEAR_PREOPEN) - return 0; - v = vfs_s_select_on_two (VFS_FILE_HANDLER_SUPER (fh)->u.fish.sockr, 0); return (((v < 0) && (errno == EINTR)) || v == 0) ? 1 : 0; @@ -1640,6 +1407,7 @@ fish_fh_new (struct vfs_s_inode *ino, gboolean changed) fh = g_new0 (fish_file_handler_t, 1); vfs_s_init_fh (VFS_FILE_HANDLER (fh), ino, changed); + fh->total = -1; /* not read yet */ return VFS_FILE_HANDLER (fh); } @@ -1651,40 +1419,45 @@ fish_fh_open (struct vfs_class *me, vfs_file_handler_t * fh, int flags, mode_t m { fish_file_handler_t *fish = FISH_FILE_HANDLER (fh); + (void) me; (void) mode; /* File will be written only, so no need to retrieve it */ - if (((flags & O_WRONLY) == O_WRONLY) && ((flags & (O_RDONLY | O_RDWR)) == 0)) + if ((flags & O_WRONLY) != 0 && (flags & (O_RDONLY | O_RDWR)) == 0) { + fh->pos = 0; + /* user pressed the button [ Append ] in the "Copy" dialog */ if ((flags & O_APPEND) != 0) - fish->append = TRUE; - - if (fh->ino->localname == NULL) { - vfs_path_t *vpath; - int tmp_handle; - - tmp_handle = vfs_mkstemps (&vpath, me->name, fh->ino->ent->name); - if (tmp_handle == -1) - { - vfs_path_free (vpath); - goto fail; - } - fh->ino->localname = g_strdup (vfs_path_as_str (vpath)); - vfs_path_free (vpath); - close (tmp_handle); + fh->pos = fh->ino->st.st_size; /* FIXME */ + fish->append = TRUE; } - return 0; } - if (fh->ino->localname == NULL && vfs_s_retrieve_file (me, fh->ino) == -1) - goto fail; - if (fh->ino->localname == NULL) - vfs_die ("retrieve_file failed to fill in localname"); - return 0; - fail: - return -1; + return 0; +} + +/* --------------------------------------------------------------------------------------------- */ + +static void +fish_fh_free (vfs_file_handler_t * fh) +{ + g_free (FISH_FILE_HANDLER (fh)->file_name); +} + +/* --------------------------------------------------------------------------------------------- */ + +static char * +fish_fh_create_filename (struct vfs_class *me, vfs_file_handler_t * fh) +{ + char *name, *quoted_name; + + name = vfs_s_fullpath (me, fh->ino); + quoted_name = strutils_shell_escape (name); + g_free (name); + + return quoted_name; } /* --------------------------------------------------------------------------------------------- */ @@ -1741,6 +1514,98 @@ fish_open (const vfs_path_t * vpath, int flags, mode_t mode) return vfs_s_open (vpath, flags, mode); } +/* --------------------------------------------------------------------------------------------- */ + +static ssize_t +fish_read (void *fh, char *buffer, size_t len) +{ + vfs_file_handler_t *file = VFS_FILE_HANDLER (fh); + struct vfs_s_super *super = VFS_FILE_HANDLER_SUPER (fh); + struct vfs_class *me = super->me; + + fish_file_handler_t *fish_file = FISH_FILE_HANDLER (fh); + fish_super_t *fish_super = FISH_SUPER (super); + + ssize_t n; + + if (fish_file->file_name == NULL) + fish_file->file_name = fish_fh_create_filename (me, file); + + n = fish_command_v (me, super, WANT_STRING, fish_super->scr_get, + "FISH_FILENAME=%s FISH_STARTOFFSET=%" PRIuMAX " FILE_CHUNKSIZE=%z;\n", + fish_file->file_name, (uintmax_t) file->pos, len); + + if (n != PRELIM) + ERRNOR (E_REMOTE, -1); + + /* get the size of remote file at rirst read */ + if (fish_file->total < 0) + { + off_t answer; + +#if SIZEOF_OFF_T == SIZEOF_LONG + answer = (off_t) strtol (reply_str, NULL, 10); +#else + answer = (off_t) g_ascii_strtoll (reply_str, NULL, 10); +#endif + fish_file->total = answer; + } + + /* define the size of chunk to be read */ + len = MIN (len, fish_file->total - file->pos); + /* get data */ + n = read (fish_super->sockr, buffer, len); + + if (n > 0) + file->pos += n; + else if (n == 0 && fish_get_reply (me, fish_super->sockr, NULL, 0) != COMPLETE) + /* file has been read completely; get reply */ + ERRNOR (E_REMOTE, -1); + + ERRNOR (errno, n); +} + +/* --------------------------------------------------------------------------------------------- */ + +static ssize_t +fish_write (void *fh, const char *buffer, size_t len) +{ + vfs_file_handler_t *file = VFS_FILE_HANDLER (fh); + struct vfs_s_super *super = VFS_FILE_HANDLER_SUPER (fh); + struct vfs_class *me = super->me; + + fish_file_handler_t *fish_file = FISH_FILE_HANDLER (fh); + fish_super_t *fish_super = FISH_SUPER (super); + + ssize_t n; + + if (fish_file->file_name == NULL) + fish_file->file_name = fish_fh_create_filename (me, file); + + n = fish_command_v (me, super, WAIT_REPLY, + fish_file->append ? fish_super->scr_append : fish_super->scr_send, + "FISH_FILENAME=%s FISH_FILESIZE=%z;\n", fish_file->file_name, len); + + if (n != PRELIM) + ERRNOR (E_REMOTE, -1); + + n = write (fish_super->sockw, buffer, len); + if (n != (ssize_t) len) + { + me->verrno = n == -1 ? errno : EIO; + ERRNOR (me->verrno, -1); + } + + if (fish_get_reply (me, fish_super->sockr, NULL, 0) != COMPLETE) + ERRNOR (E_REMOTE, -1); + + /* once wrote to new file, then append */ + fish_file->append = TRUE; + file->pos += n; + + ERRNOR (errno, n); +} + /* --------------------------------------------------------------------------------------------- */ /*** public functions ****************************************************************************/ /* --------------------------------------------------------------------------------------------- */ @@ -1750,7 +1615,7 @@ vfs_init_fish (void) { tcp_init (); - vfs_init_subclass (&fish_subclass, "fish", VFSF_REMOTE | VFSF_USETMP, "sh"); + vfs_init_subclass (&fish_subclass, "fish", VFSF_REMOTE, "sh"); vfs_fish_ops->fill_names = fish_fill_names; vfs_fish_ops->stat = fish_stat; vfs_fish_ops->lstat = fish_lstat; @@ -1759,6 +1624,8 @@ vfs_init_fish (void) vfs_fish_ops->chown = fish_chown; vfs_fish_ops->utime = fish_utime; vfs_fish_ops->open = fish_open; + vfs_fish_ops->read = fish_read; + vfs_fish_ops->write = fish_write; vfs_fish_ops->symlink = fish_symlink; vfs_fish_ops->link = fish_link; vfs_fish_ops->unlink = fish_unlink; @@ -1772,11 +1639,8 @@ vfs_init_fish (void) fish_subclass.free_archive = fish_free_archive; fish_subclass.fh_new = fish_fh_new; fish_subclass.fh_open = fish_fh_open; + fish_subclass.fh_free = fish_fh_free; fish_subclass.dir_load = fish_dir_load; - fish_subclass.file_store = fish_file_store; - fish_subclass.linear_start = fish_linear_start; - fish_subclass.linear_read = fish_linear_read; - fish_subclass.linear_close = fish_linear_close; vfs_register_class (vfs_fish_ops); } diff --git a/src/vfs/fish/fishdef.h b/src/vfs/fish/fishdef.h index 1bb836bb3..b79b5ad14 100644 --- a/src/vfs/fish/fishdef.h +++ b/src/vfs/fish/fishdef.h @@ -129,15 +129,29 @@ /* default 'retr' script */ #define FISH_GET_DEF_CONTENT "" \ +"FILENAME=\"/${FISH_FILENAME}\"\n" \ +"STARTOFFSET=${FISH_STARTOFFSET}\n" \ +"CHUNKSIZE=${FISH_CHUNKSIZE}\n" \ "export LC_TIME=C\n" \ -"#RETR $FISH_FILENAME\n" \ -"if dd if=\"/${FISH_FILENAME}\" of=/dev/null bs=1 count=1 2>/dev/null ; then\n" \ -" ls -ln \"/${FISH_FILENAME}\" 2>/dev/null | (\n" \ +"#RETR $FILENAME $STARTOFFSET $CHUNKSIZE\n" \ +"if dd if=\"${FILENAME}\" of=/dev/null bs=1 count=1 2>/dev/null ; then\n" \ +" file_size=`ls -ln \"${FILENAME}\" 2>/dev/null | (\n" \ " read p l u g s r\n" \ " echo $s\n" \ -" )\n" \ +" )`\n" \ +" echo ${file_size}" \ " echo \"### 100\"\n" \ -" cat \"/${FISH_FILENAME}\"\n" \ +" next_offset=`expr ${STARTOFFSET} + ${CHUNKSIZE}`\n" \ +" if [ ${next_offset} -gt ${file_size} ]; then\n" \ +" CHUNKSIZE=`expr ${file_size} - ${STARTOFFSET}`\n" \ +" fi\n" \ +" if [ ${CHUNKSIZE} -eq 0 ]; then\n" \ +" echo -n\n" \ +" elsif [ ${STARTOFFSET} -eq 0 ]; then\n" \ +" dd if=\"${FILENAME}\" bs=${CHUNKSIZE} 2>/dev/null\n" \ +" else\n" \ +" dd if=\"${FILENAME}\" ibs=${STARTOFFSET} skip=1" \ +" obs=${CHUNKSIZE} count=1\n" \ " echo \"### 200\"\n" \ "else\n" \ " echo \"### 500\"\n" \ diff --git a/src/vfs/fish/helpers/get b/src/vfs/fish/helpers/get index 762267afd..ec2052b0e 100644 --- a/src/vfs/fish/helpers/get +++ b/src/vfs/fish/helpers/get @@ -1,30 +1,35 @@ -#RETR $FISH_FILENAME $FISH_START_OFFSET +#RETR $FISH_FILENAME $FISH_START_OFFSET $FISH_CHUNK_SIZE LC_TIME=C export LC_TIME fish_get_perl () { FILENAME=$1 -OFFSET=$2 +STARTOFFSET=$2 +CHUNKSIZE=$3 perl -e ' use strict; use POSIX; use Fcntl; my $filename = $ARGV[0]; my $pos = $ARGV[1]; +my $chunk = $ARGV[2]; my $content; +my $next_pos; my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = lstat("$filename"); my $n; if (open IFILE,$filename) { - if ($size<$pos) { - printf("0\n"); - } else { - $size-=$pos; - printf("$size\n"); - } + printf("$size\n"); printf("### 100\n"); - seek (IFILE, $pos, 0); - while ($n = read(IFILE,$content,$blksize)!= 0) { - print $content; + $next_pos = $pos + $chunk; + if ($next_pos > $size) { + $chunk = $size - $pos; + } + if ($chunk != 0) { + seek (IFILE, $pos, 0); + $n = read(IFILE,$content,$chunksize); + if ($n != 0) { + print $content; + } } close IFILE; printf("### 200\n"); @@ -32,13 +37,14 @@ if (open IFILE,$filename) { printf("### 500\n"); } exit 0 -' "${FILENAME}" $OFFSET +' "${FILENAME}" $STARTOFFSET $CHUNKSIZE } -fish_get_tail () +fish_get_head_tail () { FILENAME=$1 -OFFSET=$2 +STARTOFFSET=$2 +CHUNKSIZE=$3 LC_TIME=C export LC_TIME if dd if="${FILENAME}" of=/dev/null bs=1 count=1 2>/dev/null ; then @@ -46,20 +52,19 @@ if dd if="${FILENAME}" of=/dev/null bs=1 count=1 2>/dev/null ; then read p l u g s r echo $s )` - if [ $OFFSET -gt 0 ]; then - file_size=`expr $file_size - $OFFSET` - OFFSET=`expr $OFFSET + 1` - fi - if [ $file_size -gt 0 ]; then - echo $file_size - else - echo 0 - fi + echo ${file_size} echo "### 100" - if [ $OFFSET -gt 0 ]; then - tail -c +${OFFSET} "${FILENAME}" + next_offset=`expr ${STARTOFFSET} + ${CHUNKSIZE}` + if [ ${next_offset} -gt ${file_size} ]; then + CHUNKSIZE=`expr ${file_size} - ${STARTOFFSET}` + next_offset=${file_size} + fi + if [ ${CHUNKSIZE} -eq 0 ]; then + echo -n + elif [ ${STARTOFFSET} -eq 0 ]; then + head -q -c ${CHUNKSIZE} "${FILENAME}" else - cat "${FILENAME}" + head -q -c $next_offset "${FILENAME}" | tail -q -c +${CHUNKSIZE} - fi echo "### 200" else @@ -70,7 +75,8 @@ fi fish_get_dd () { FILENAME=$1 -OFFSET=$2 +STARTOFFSET=$2 +CHUNKSIZE=$3 LC_TIME=C export LC_TIME if dd if="${FILENAME}" of=/dev/null bs=1 count=1 2>/dev/null ; then @@ -78,17 +84,18 @@ if dd if="${FILENAME}" of=/dev/null bs=1 count=1 2>/dev/null ; then read p l u g s r echo $s )` - file_size=`expr $file_size - $OFFSET` - if [ $file_size -gt 0 ]; then - echo $file_size - else - echo 0 - fi + echo ${file_size} echo "### 100" - if [ $OFFSET -gt 0 ]; then - dd skip=$OFFSET ibs=1 if="${FILENAME}" 2>/dev/null + next_offset=`expr ${STARTOFFSET} + ${CHUNKSIZE}` + if [ ${next_offset} -gt ${file_size} ]; then + CHUNKSIZE=`expr ${file_size} - ${STARTOFFSET}` + fi + if [ ${CHUNKSIZE} -eq 0 ]; then + echo -n + elif [ ${STARTOFFSET} -eq 0 ]; then + dd if="${FILENAME}" bs=${CHUNKSIZE} 2>/dev/null else - cat "${FILENAME}" + dd if="${FILENAME}" ibs=${STARTOFFSET} skip=1 obs=${CHUNKSIZE} count=1 2>/dev/null fi echo "### 200" else @@ -97,9 +104,9 @@ fi } if [ -n "${FISH_HAVE_PERL}" ]; then - fish_get_perl "/${FISH_FILENAME}" ${FISH_START_OFFSET} -elif [ -n "${FISH_HAVE_TAIL}" ]; then - fish_get_tail "/${FISH_FILENAME}" ${FISH_START_OFFSET} + fish_get_perl "/${FISH_FILENAME}" ${FISH_STARTOFFSET} ${FISH_CHUNKSIZE} +elif [ -n "${FISH_HAVE_HEAD}" -a -n "${FISH_HAVE_TAIL}" ]; then + fish_get_head_tail "/${FISH_FILENAME}" ${FISH_STARTOFFSET} ${FISH_CHUNKSIZE} else - fish_get_dd "/${FISH_FILENAME}" ${FISH_START_OFFSET} + fish_get_dd "/${FISH_FILENAME}" ${FISH_STARTOFFSET} ${FISH_CHUNKSIZE} fi