Ticket #77: fish: get rid of copy file to tmp before transfer [travis ci skip].

NOT FINISHED!

Signed-off-by: Andrew Borodin <aborodin@vmail.ru>
This commit is contained in:
Andrew Borodin 2019-04-02 14:14:10 +03:00
parent ba28de595d
commit a18a7fa396
3 changed files with 193 additions and 308 deletions

View File

@ -151,7 +151,7 @@ typedef struct
{
vfs_file_handler_t base; /* base class */
off_t got;
char *file_name;
off_t total;
gboolean append;
} fish_file_handler_t;
@ -962,233 +962,6 @@ fish_dir_load (struct vfs_class *me, struct vfs_s_inode *dir, char *remote_path)
/* --------------------------------------------------------------------------------------------- */
static int
fish_file_store (struct vfs_class *me, vfs_file_handler_t * fh, char *name, char *localname)
{
fish_file_handler_t *fish = FISH_FILE_HANDLER (fh);
struct vfs_s_super *super = VFS_FILE_HANDLER_SUPER (fh);
fish_super_t *fish_super = FISH_SUPER (super);
int code;
off_t total = 0;
char buffer[BUF_8K];
struct stat s;
int h;
char *quoted_name;
h = open (localname, O_RDONLY);
if (h == -1)
ERRNOR (EIO, -1);
if (fstat (h, &s) < 0)
{
close (h);
ERRNOR (EIO, -1);
}
/* First, try this as stor:
*
* ( head -c number ) | ( cat > file; cat >/dev/null )
*
* If 'head' is not present on the remote system, 'dd' will be used.
* Unfortunately, we cannot trust most non-GNU 'head' implementations
* even if '-c' options is supported. Therefore, we separate GNU head
* (and other modern heads?) using '-q' and '-' . This causes another
* implementations to fail (because of "incorrect options").
*
* Fallback is:
*
* rest=<number>
* while [ $rest -gt 0 ]
* do
* cnt=`expr \( $rest + 255 \) / 256`
* n=`dd bs=256 count=$cnt | tee -a <target_file> | wc -c`
* rest=`expr $rest - $n`
* done
*
* 'dd' was not designed for full filling of input buffers,
* and does not report exact number of bytes (not blocks).
* Therefore a more complex shell script is needed.
*
* On some systems non-GNU head writes "Usage:" error report to stdout
* instead of stderr. It makes impossible the use of "head || dd"
* algorithm for file appending case, therefore just "dd" is used for it.
*/
quoted_name = strutils_shell_escape (name);
vfs_print_message (_("fish: store %s: sending command..."), quoted_name);
/* FIXME: File size is limited to ULONG_MAX */
code =
fish_command_v (me, super, WAIT_REPLY,
fish->append ? fish_super->scr_append : fish_super->scr_send,
"FISH_FILENAME=%s FISH_FILESIZE=%" PRIuMAX ";\n", quoted_name,
(uintmax_t) s.st_size);
g_free (quoted_name);
if (code != PRELIM)
{
close (h);
ERRNOR (E_REMOTE, -1);
}
while (TRUE)
{
ssize_t n, t;
while ((n = read (h, buffer, sizeof (buffer))) < 0)
{
if ((errno == EINTR) && tty_got_interrupt ())
continue;
vfs_print_message ("%s", _("fish: Local read failed, sending zeros"));
close (h);
h = open ("/dev/zero", O_RDONLY);
}
if (n == 0)
break;
t = write (fish_super->sockw, buffer, n);
if (t != n)
{
if (t == -1)
me->verrno = errno;
else
me->verrno = EIO;
goto error_return;
}
tty_disable_interrupt_key ();
total += n;
vfs_print_message ("%s: %" PRIuMAX "/%" PRIuMAX, _("fish: storing file"),
(uintmax_t) total, (uintmax_t) s.st_size);
}
close (h);
if (fish_get_reply (me, fish_super->sockr, NULL, 0) != COMPLETE)
ERRNOR (E_REMOTE, -1);
return 0;
error_return:
close (h);
fish_get_reply (me, fish_super->sockr, NULL, 0);
return -1;
}
/* --------------------------------------------------------------------------------------------- */
static int
fish_linear_start (struct vfs_class *me, vfs_file_handler_t * fh, off_t offset)
{
fish_file_handler_t *fish = FISH_FILE_HANDLER (fh);
struct vfs_s_super *super = VFS_FILE_HANDLER_SUPER (fh);
char *name;
char *quoted_name;
name = vfs_s_fullpath (me, fh->ino);
if (name == NULL)
return 0;
quoted_name = strutils_shell_escape (name);
g_free (name);
fish->append = FALSE;
/*
* Check whether the remote file is readable by using 'dd' to copy
* a single byte from the remote file to /dev/null. If 'dd' completes
* with exit status of 0 use 'cat' to send the file contents to the
* standard output (i.e. over the network).
*/
offset =
fish_command_v (me, super, WANT_STRING, FISH_SUPER (super)->scr_get,
"FISH_FILENAME=%s FISH_START_OFFSET=%" PRIuMAX ";\n", quoted_name,
(uintmax_t) offset);
g_free (quoted_name);
if (offset != PRELIM)
ERRNOR (E_REMOTE, 0);
fh->linear = LS_LINEAR_OPEN;
fish->got = 0;
errno = 0;
#if SIZEOF_OFF_T == SIZEOF_LONG
fish->total = (off_t) strtol (reply_str, NULL, 10);
#else
fish->total = (off_t) g_ascii_strtoll (reply_str, NULL, 10);
#endif
if (errno != 0)
ERRNOR (E_REMOTE, 0);
return 1;
}
/* --------------------------------------------------------------------------------------------- */
static void
fish_linear_abort (struct vfs_class *me, vfs_file_handler_t * fh)
{
fish_file_handler_t *fish = FISH_FILE_HANDLER (fh);
struct vfs_s_super *super = VFS_FILE_HANDLER_SUPER (fh);
char buffer[BUF_8K];
ssize_t n;
vfs_print_message ("%s", _("Aborting transfer..."));
do
{
n = MIN ((off_t) sizeof (buffer), (fish->total - fish->got));
if (n != 0)
{
n = read (FISH_SUPER (super)->sockr, buffer, n);
if (n < 0)
return;
fish->got += n;
}
}
while (n != 0);
if (fish_get_reply (me, FISH_SUPER (super)->sockr, NULL, 0) != COMPLETE)
vfs_print_message ("%s", _("Error reported after abort."));
else
vfs_print_message ("%s", _("Aborted transfer would be successful."));
}
/* --------------------------------------------------------------------------------------------- */
static ssize_t
fish_linear_read (struct vfs_class *me, vfs_file_handler_t * fh, void *buf, size_t len)
{
fish_file_handler_t *fish = FISH_FILE_HANDLER (fh);
struct vfs_s_super *super = VFS_FILE_HANDLER_SUPER (fh);
ssize_t n = 0;
len = MIN ((size_t) (fish->total - fish->got), len);
tty_disable_interrupt_key ();
while (len != 0 && ((n = read (FISH_SUPER (super)->sockr, buf, len)) < 0))
{
if ((errno == EINTR) && !tty_got_interrupt ())
continue;
break;
}
tty_enable_interrupt_key ();
if (n > 0)
fish->got += n;
else if (n < 0)
fish_linear_abort (me, fh);
else if (fish_get_reply (me, FISH_SUPER (super)->sockr, NULL, 0) != COMPLETE)
ERRNOR (E_REMOTE, -1);
ERRNOR (errno, n);
}
/* --------------------------------------------------------------------------------------------- */
static void
fish_linear_close (struct vfs_class *me, vfs_file_handler_t * fh)
{
fish_file_handler_t *fish = FISH_FILE_HANDLER (fh);
if (fish->total != fish->got)
fish_linear_abort (me, fh);
}
/* --------------------------------------------------------------------------------------------- */
static int
fish_ctl (void *fh, int ctlop, void *arg)
{
@ -1203,14 +976,8 @@ fish_ctl (void *fh, int ctlop, void *arg)
{
case VFS_CTL_IS_NOTREADY:
{
vfs_file_handler_t *file = VFS_FILE_HANDLER (fh);
int v;
if (file->linear == LS_NOT_LINEAR)
vfs_die ("You may not do this");
if (file->linear == LS_LINEAR_CLOSED || file->linear == LS_LINEAR_PREOPEN)
return 0;
v = vfs_s_select_on_two (VFS_FILE_HANDLER_SUPER (fh)->u.fish.sockr, 0);
return (((v < 0) && (errno == EINTR)) || v == 0) ? 1 : 0;
@ -1640,6 +1407,7 @@ fish_fh_new (struct vfs_s_inode *ino, gboolean changed)
fh = g_new0 (fish_file_handler_t, 1);
vfs_s_init_fh (VFS_FILE_HANDLER (fh), ino, changed);
fh->total = -1; /* not read yet */
return VFS_FILE_HANDLER (fh);
}
@ -1651,40 +1419,45 @@ fish_fh_open (struct vfs_class *me, vfs_file_handler_t * fh, int flags, mode_t m
{
fish_file_handler_t *fish = FISH_FILE_HANDLER (fh);
(void) me;
(void) mode;
/* File will be written only, so no need to retrieve it */
if (((flags & O_WRONLY) == O_WRONLY) && ((flags & (O_RDONLY | O_RDWR)) == 0))
if ((flags & O_WRONLY) != 0 && (flags & (O_RDONLY | O_RDWR)) == 0)
{
fh->pos = 0;
/* user pressed the button [ Append ] in the "Copy" dialog */
if ((flags & O_APPEND) != 0)
{
fh->pos = fh->ino->st.st_size; /* FIXME */
fish->append = TRUE;
if (fh->ino->localname == NULL)
{
vfs_path_t *vpath;
int tmp_handle;
tmp_handle = vfs_mkstemps (&vpath, me->name, fh->ino->ent->name);
if (tmp_handle == -1)
{
vfs_path_free (vpath);
goto fail;
}
fh->ino->localname = g_strdup (vfs_path_as_str (vpath));
vfs_path_free (vpath);
close (tmp_handle);
}
return 0;
}
if (fh->ino->localname == NULL && vfs_s_retrieve_file (me, fh->ino) == -1)
goto fail;
if (fh->ino->localname == NULL)
vfs_die ("retrieve_file failed to fill in localname");
return 0;
fail:
return -1;
/* --------------------------------------------------------------------------------------------- */
static void
fish_fh_free (vfs_file_handler_t * fh)
{
g_free (FISH_FILE_HANDLER (fh)->file_name);
}
/* --------------------------------------------------------------------------------------------- */
static char *
fish_fh_create_filename (struct vfs_class *me, vfs_file_handler_t * fh)
{
char *name, *quoted_name;
name = vfs_s_fullpath (me, fh->ino);
quoted_name = strutils_shell_escape (name);
g_free (name);
return quoted_name;
}
/* --------------------------------------------------------------------------------------------- */
@ -1741,6 +1514,98 @@ fish_open (const vfs_path_t * vpath, int flags, mode_t mode)
return vfs_s_open (vpath, flags, mode);
}
/* --------------------------------------------------------------------------------------------- */
static ssize_t
fish_read (void *fh, char *buffer, size_t len)
{
vfs_file_handler_t *file = VFS_FILE_HANDLER (fh);
struct vfs_s_super *super = VFS_FILE_HANDLER_SUPER (fh);
struct vfs_class *me = super->me;
fish_file_handler_t *fish_file = FISH_FILE_HANDLER (fh);
fish_super_t *fish_super = FISH_SUPER (super);
ssize_t n;
if (fish_file->file_name == NULL)
fish_file->file_name = fish_fh_create_filename (me, file);
n = fish_command_v (me, super, WANT_STRING, fish_super->scr_get,
"FISH_FILENAME=%s FISH_STARTOFFSET=%" PRIuMAX " FILE_CHUNKSIZE=%z;\n",
fish_file->file_name, (uintmax_t) file->pos, len);
if (n != PRELIM)
ERRNOR (E_REMOTE, -1);
/* get the size of remote file at rirst read */
if (fish_file->total < 0)
{
off_t answer;
#if SIZEOF_OFF_T == SIZEOF_LONG
answer = (off_t) strtol (reply_str, NULL, 10);
#else
answer = (off_t) g_ascii_strtoll (reply_str, NULL, 10);
#endif
fish_file->total = answer;
}
/* define the size of chunk to be read */
len = MIN (len, fish_file->total - file->pos);
/* get data */
n = read (fish_super->sockr, buffer, len);
if (n > 0)
file->pos += n;
else if (n == 0 && fish_get_reply (me, fish_super->sockr, NULL, 0) != COMPLETE)
/* file has been read completely; get reply */
ERRNOR (E_REMOTE, -1);
ERRNOR (errno, n);
}
/* --------------------------------------------------------------------------------------------- */
static ssize_t
fish_write (void *fh, const char *buffer, size_t len)
{
vfs_file_handler_t *file = VFS_FILE_HANDLER (fh);
struct vfs_s_super *super = VFS_FILE_HANDLER_SUPER (fh);
struct vfs_class *me = super->me;
fish_file_handler_t *fish_file = FISH_FILE_HANDLER (fh);
fish_super_t *fish_super = FISH_SUPER (super);
ssize_t n;
if (fish_file->file_name == NULL)
fish_file->file_name = fish_fh_create_filename (me, file);
n = fish_command_v (me, super, WAIT_REPLY,
fish_file->append ? fish_super->scr_append : fish_super->scr_send,
"FISH_FILENAME=%s FISH_FILESIZE=%z;\n", fish_file->file_name, len);
if (n != PRELIM)
ERRNOR (E_REMOTE, -1);
n = write (fish_super->sockw, buffer, len);
if (n != (ssize_t) len)
{
me->verrno = n == -1 ? errno : EIO;
ERRNOR (me->verrno, -1);
}
if (fish_get_reply (me, fish_super->sockr, NULL, 0) != COMPLETE)
ERRNOR (E_REMOTE, -1);
/* once wrote to new file, then append */
fish_file->append = TRUE;
file->pos += n;
ERRNOR (errno, n);
}
/* --------------------------------------------------------------------------------------------- */
/*** public functions ****************************************************************************/
/* --------------------------------------------------------------------------------------------- */
@ -1750,7 +1615,7 @@ vfs_init_fish (void)
{
tcp_init ();
vfs_init_subclass (&fish_subclass, "fish", VFSF_REMOTE | VFSF_USETMP, "sh");
vfs_init_subclass (&fish_subclass, "fish", VFSF_REMOTE, "sh");
vfs_fish_ops->fill_names = fish_fill_names;
vfs_fish_ops->stat = fish_stat;
vfs_fish_ops->lstat = fish_lstat;
@ -1759,6 +1624,8 @@ vfs_init_fish (void)
vfs_fish_ops->chown = fish_chown;
vfs_fish_ops->utime = fish_utime;
vfs_fish_ops->open = fish_open;
vfs_fish_ops->read = fish_read;
vfs_fish_ops->write = fish_write;
vfs_fish_ops->symlink = fish_symlink;
vfs_fish_ops->link = fish_link;
vfs_fish_ops->unlink = fish_unlink;
@ -1772,11 +1639,8 @@ vfs_init_fish (void)
fish_subclass.free_archive = fish_free_archive;
fish_subclass.fh_new = fish_fh_new;
fish_subclass.fh_open = fish_fh_open;
fish_subclass.fh_free = fish_fh_free;
fish_subclass.dir_load = fish_dir_load;
fish_subclass.file_store = fish_file_store;
fish_subclass.linear_start = fish_linear_start;
fish_subclass.linear_read = fish_linear_read;
fish_subclass.linear_close = fish_linear_close;
vfs_register_class (vfs_fish_ops);
}

View File

@ -129,15 +129,29 @@
/* default 'retr' script */
#define FISH_GET_DEF_CONTENT "" \
"FILENAME=\"/${FISH_FILENAME}\"\n" \
"STARTOFFSET=${FISH_STARTOFFSET}\n" \
"CHUNKSIZE=${FISH_CHUNKSIZE}\n" \
"export LC_TIME=C\n" \
"#RETR $FISH_FILENAME\n" \
"if dd if=\"/${FISH_FILENAME}\" of=/dev/null bs=1 count=1 2>/dev/null ; then\n" \
" ls -ln \"/${FISH_FILENAME}\" 2>/dev/null | (\n" \
"#RETR $FILENAME $STARTOFFSET $CHUNKSIZE\n" \
"if dd if=\"${FILENAME}\" of=/dev/null bs=1 count=1 2>/dev/null ; then\n" \
" file_size=`ls -ln \"${FILENAME}\" 2>/dev/null | (\n" \
" read p l u g s r\n" \
" echo $s\n" \
" )\n" \
" )`\n" \
" echo ${file_size}" \
" echo \"### 100\"\n" \
" cat \"/${FISH_FILENAME}\"\n" \
" next_offset=`expr ${STARTOFFSET} + ${CHUNKSIZE}`\n" \
" if [ ${next_offset} -gt ${file_size} ]; then\n" \
" CHUNKSIZE=`expr ${file_size} - ${STARTOFFSET}`\n" \
" fi\n" \
" if [ ${CHUNKSIZE} -eq 0 ]; then\n" \
" echo -n\n" \
" elsif [ ${STARTOFFSET} -eq 0 ]; then\n" \
" dd if=\"${FILENAME}\" bs=${CHUNKSIZE} 2>/dev/null\n" \
" else\n" \
" dd if=\"${FILENAME}\" ibs=${STARTOFFSET} skip=1" \
" obs=${CHUNKSIZE} count=1\n" \
" echo \"### 200\"\n" \
"else\n" \
" echo \"### 500\"\n" \

View File

@ -1,44 +1,50 @@
#RETR $FISH_FILENAME $FISH_START_OFFSET
#RETR $FISH_FILENAME $FISH_START_OFFSET $FISH_CHUNK_SIZE
LC_TIME=C
export LC_TIME
fish_get_perl ()
{
FILENAME=$1
OFFSET=$2
STARTOFFSET=$2
CHUNKSIZE=$3
perl -e '
use strict;
use POSIX;
use Fcntl;
my $filename = $ARGV[0];
my $pos = $ARGV[1];
my $chunk = $ARGV[2];
my $content;
my $next_pos;
my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = lstat("$filename");
my $n;
if (open IFILE,$filename) {
if ($size<$pos) {
printf("0\n");
} else {
$size-=$pos;
printf("$size\n");
}
printf("### 100\n");
$next_pos = $pos + $chunk;
if ($next_pos > $size) {
$chunk = $size - $pos;
}
if ($chunk != 0) {
seek (IFILE, $pos, 0);
while ($n = read(IFILE,$content,$blksize)!= 0) {
$n = read(IFILE,$content,$chunksize);
if ($n != 0) {
print $content;
}
}
close IFILE;
printf("### 200\n");
} else {
printf("### 500\n");
}
exit 0
' "${FILENAME}" $OFFSET
' "${FILENAME}" $STARTOFFSET $CHUNKSIZE
}
fish_get_tail ()
fish_get_head_tail ()
{
FILENAME=$1
OFFSET=$2
STARTOFFSET=$2
CHUNKSIZE=$3
LC_TIME=C
export LC_TIME
if dd if="${FILENAME}" of=/dev/null bs=1 count=1 2>/dev/null ; then
@ -46,20 +52,19 @@ if dd if="${FILENAME}" of=/dev/null bs=1 count=1 2>/dev/null ; then
read p l u g s r
echo $s
)`
if [ $OFFSET -gt 0 ]; then
file_size=`expr $file_size - $OFFSET`
OFFSET=`expr $OFFSET + 1`
fi
if [ $file_size -gt 0 ]; then
echo $file_size
else
echo 0
fi
echo ${file_size}
echo "### 100"
if [ $OFFSET -gt 0 ]; then
tail -c +${OFFSET} "${FILENAME}"
next_offset=`expr ${STARTOFFSET} + ${CHUNKSIZE}`
if [ ${next_offset} -gt ${file_size} ]; then
CHUNKSIZE=`expr ${file_size} - ${STARTOFFSET}`
next_offset=${file_size}
fi
if [ ${CHUNKSIZE} -eq 0 ]; then
echo -n
elif [ ${STARTOFFSET} -eq 0 ]; then
head -q -c ${CHUNKSIZE} "${FILENAME}"
else
cat "${FILENAME}"
head -q -c $next_offset "${FILENAME}" | tail -q -c +${CHUNKSIZE} -
fi
echo "### 200"
else
@ -70,7 +75,8 @@ fi
fish_get_dd ()
{
FILENAME=$1
OFFSET=$2
STARTOFFSET=$2
CHUNKSIZE=$3
LC_TIME=C
export LC_TIME
if dd if="${FILENAME}" of=/dev/null bs=1 count=1 2>/dev/null ; then
@ -78,17 +84,18 @@ if dd if="${FILENAME}" of=/dev/null bs=1 count=1 2>/dev/null ; then
read p l u g s r
echo $s
)`
file_size=`expr $file_size - $OFFSET`
if [ $file_size -gt 0 ]; then
echo $file_size
else
echo 0
fi
echo ${file_size}
echo "### 100"
if [ $OFFSET -gt 0 ]; then
dd skip=$OFFSET ibs=1 if="${FILENAME}" 2>/dev/null
next_offset=`expr ${STARTOFFSET} + ${CHUNKSIZE}`
if [ ${next_offset} -gt ${file_size} ]; then
CHUNKSIZE=`expr ${file_size} - ${STARTOFFSET}`
fi
if [ ${CHUNKSIZE} -eq 0 ]; then
echo -n
elif [ ${STARTOFFSET} -eq 0 ]; then
dd if="${FILENAME}" bs=${CHUNKSIZE} 2>/dev/null
else
cat "${FILENAME}"
dd if="${FILENAME}" ibs=${STARTOFFSET} skip=1 obs=${CHUNKSIZE} count=1 2>/dev/null
fi
echo "### 200"
else
@ -97,9 +104,9 @@ fi
}
if [ -n "${FISH_HAVE_PERL}" ]; then
fish_get_perl "/${FISH_FILENAME}" ${FISH_START_OFFSET}
elif [ -n "${FISH_HAVE_TAIL}" ]; then
fish_get_tail "/${FISH_FILENAME}" ${FISH_START_OFFSET}
fish_get_perl "/${FISH_FILENAME}" ${FISH_STARTOFFSET} ${FISH_CHUNKSIZE}
elif [ -n "${FISH_HAVE_HEAD}" -a -n "${FISH_HAVE_TAIL}" ]; then
fish_get_head_tail "/${FISH_FILENAME}" ${FISH_STARTOFFSET} ${FISH_CHUNKSIZE}
else
fish_get_dd "/${FISH_FILENAME}" ${FISH_START_OFFSET}
fish_get_dd "/${FISH_FILENAME}" ${FISH_STARTOFFSET} ${FISH_CHUNKSIZE}
fi