From bbcfda3c629a8d49f8990b48d2f6e7eb6d114959 Mon Sep 17 00:00:00 2001 From: slowpeek Date: Fri, 26 Apr 2024 01:24:56 +0300 Subject: [PATCH 1/3] Ticket #3570: reimplement xorriso part of iso9660 listing. When xorriso is installed, iso9660 listing was done with `xorriso .. -lsl ..` per dir recursively. It was unbearably slow on iso images with lots of subdirs. For example, it took my machine 2 minutes to open debian-12.5.0-amd64-DVD-1.iso which contains 2627 dirs. This commit makes use of `xorriso .. -find / -exec lsdl ..` instead to list the whole directory tree in a single pass. Also, parsing of xorriso output has been reworked and as a result such previously missing items became visible in the listing: - symlinks - the boot catalog file Closes MidnightCommander/mc#196. Signed-off-by: Andrew Borodin --- src/vfs/extfs/helpers/iso9660.in | 80 ++++++++++++++++++++++++++------ 1 file changed, 65 insertions(+), 15 deletions(-) diff --git a/src/vfs/extfs/helpers/iso9660.in b/src/vfs/extfs/helpers/iso9660.in index f78db61cd..1b0532c7a 100644 --- a/src/vfs/extfs/helpers/iso9660.in +++ b/src/vfs/extfs/helpers/iso9660.in @@ -11,6 +11,7 @@ # Kachalov Anton , 2003 # Victor Ananjevsky , 2013 # slava zanko , 2013 +# slowpeek , 2024 # # This file is part of the Midnight Commander. # @@ -33,27 +34,76 @@ XORRISO=$(which xorriso 2>/dev/null) +# This snippet is used to undo xorriso's Text_shellsafe(). Pseudocode for the +# function: +# +# Text_shellsafe( s) { +# return q(') replace(q('), q('"'"'), s) q(') +# } +# +# Xorriso_ls() uses two formats for pathes: +# - generic: Text_shellsafe(path) +# - symlink: Text_shellsafe(link) q( -> ) Text_shellsafe(target) +# +# References in xorriso sources: +# - iso_tree.c:Xorriso_ls() +# - text_io.c:Xorriso_esc_filepath() +# - misc_funct.c:Text_shellsafe() +awk_xorriso_unesc=$(cat <<'EOF' +{ + # Start of the path part + i = index($0, "'") + + s = substr($0, i) + # The path part with the wrapping quotes removed + s = substr(s, 2, length(s)-2) + + if (substr($0, 1, 1) == "l") { + s1=s + if (gsub("->", "", s1) > 1) { + # Ambiguity: either the link or its target contains q(->) in + # addition to the link/target delimiter + next + } + + j = index(s, "' -> '") + # link -> target with the wrapping quotes removed from both + s = substr(s, 1, j-1) " -> " substr(s, j+6) + } else if (index(s, "->")) { + # Ambiguity: not a symlink, but there is q(->) + next + } + + gsub("'\"'\"'", "'", s) + print substr($0, 1, i-1) s +} +EOF +) + xorriso_list() { if test -z "$XORRISO"; then return 1 fi - local dir attr ln usr gr sz dt1 dt2 dt3 nm len name lsl r - dir="${2:-/}" - lsl=$( $XORRISO -abort_on FATAL -dev stdio:"$1" -cd "$dir" -lsl 2> /dev/null ) - r=$? - test $r -gt 0 && return $r - echo "$lsl" | grep "^[-d]" | \ - while read attr ln usr gr sz dt1 dt2 dt3 nm ; do - len=$((${#nm} - 1)) - name=$(printf -- "$nm" | cut -c2-$len) # remove quotes + local temp_ls + temp_ls=$(mktemp "${MC_TMPDIR:-/tmp}"/mc-iso9660.XXXXXX) || return 1 - if test $(printf -- "$attr" | cut -c1-1) != "d"; then - printf -- "%s %s %s %s %s %s %s %s %s/%s\n" "$attr" "$ln" "$usr" "$gr" "$sz" "$dt1" "$dt2" "$dt3" "$dir" "$name" - else - xorriso_list "$1" "$dir/$name" - fi - done + "$XORRISO" -abort_on FATAL -dev stdio:"$1" -find / -exec lsdl 2>/dev/null >"$temp_ls" + local r=$? + + if [ "$r" != 0 ]; then + rm -f "$temp_ls" + return "$r" + fi + + # The first line is /, skip it + tail -n+2 "$temp_ls" | + # disk_ops.c:Xorriso_format_ls_l() prints the boot catalog file as of + # type "e". Make it a generic file + sed -E 's,^e,-,' | + @AWK@ "$awk_xorriso_unesc" + + rm -f "$temp_ls" } xorriso_copyout() { From 094a140c263c8523704395ea7632f7e3d0e9a01d Mon Sep 17 00:00:00 2001 From: slowpeek Date: Tue, 7 May 2024 18:42:24 +0300 Subject: [PATCH 2/3] extfs: modify the iso9660 extfs helper to support testing (xorriso part) Signed-off-by: Andrew Borodin --- src/vfs/extfs/helpers/iso9660.in | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/vfs/extfs/helpers/iso9660.in b/src/vfs/extfs/helpers/iso9660.in index 1b0532c7a..a4519dc3c 100644 --- a/src/vfs/extfs/helpers/iso9660.in +++ b/src/vfs/extfs/helpers/iso9660.in @@ -88,7 +88,8 @@ xorriso_list() { local temp_ls temp_ls=$(mktemp "${MC_TMPDIR:-/tmp}"/mc-iso9660.XXXXXX) || return 1 - "$XORRISO" -abort_on FATAL -dev stdio:"$1" -find / -exec lsdl 2>/dev/null >"$temp_ls" + # $XORRISO must be unquoted here to hook into the testing framework + $XORRISO -abort_on FATAL -dev stdio:"$1" -find / -exec lsdl 2>/dev/null >"$temp_ls" local r=$? if [ "$r" != 0 ]; then @@ -245,6 +246,20 @@ shift case "$cmd" in list) + if [ -n "${MC_TEST_EXTFS_LIST_CMD:-}" ]; then + case "${MC_TEST_EXTFS_ISO9660_TOOL}" in + xorriso) + XORRISO="$MC_TEST_EXTFS_LIST_CMD" + xorriso_list "$@" || exit 1 + ;; + *) + exit 1 + ;; + esac + + exit 0 + fi + xorriso_list "$@" || { test_iso "$@" || exit 1 mcisofs_list "$@" || exit 1 From 62915a38f1b4777f7ac11e1ee8e64aac20235cb8 Mon Sep 17 00:00:00 2001 From: slowpeek Date: Tue, 7 May 2024 18:48:51 +0300 Subject: [PATCH 3/3] extfs: add test data for the iso9660 extfs helper (xorriso part) Signed-off-by: Andrew Borodin --- tests/src/vfs/extfs/helpers-list/Makefile.am | 4 ++ .../helpers-list/data/iso9660.xorriso.README | 49 +++++++++++++++++++ .../data/iso9660.xorriso.env_vars | 1 + .../helpers-list/data/iso9660.xorriso.input | 35 +++++++++++++ .../helpers-list/data/iso9660.xorriso.output | 34 +++++++++++++ 5 files changed, 123 insertions(+) create mode 100644 tests/src/vfs/extfs/helpers-list/data/iso9660.xorriso.README create mode 100644 tests/src/vfs/extfs/helpers-list/data/iso9660.xorriso.env_vars create mode 100644 tests/src/vfs/extfs/helpers-list/data/iso9660.xorriso.input create mode 100644 tests/src/vfs/extfs/helpers-list/data/iso9660.xorriso.output diff --git a/tests/src/vfs/extfs/helpers-list/Makefile.am b/tests/src/vfs/extfs/helpers-list/Makefile.am index 6c6c04482..978029acf 100644 --- a/tests/src/vfs/extfs/helpers-list/Makefile.am +++ b/tests/src/vfs/extfs/helpers-list/Makefile.am @@ -32,6 +32,10 @@ data_files_to_distribute = \ data/hp48+.args \ data/hp48+.input \ data/hp48+.output \ + data/iso9660.xorriso.env_vars \ + data/iso9660.xorriso.input \ + data/iso9660.xorriso.output \ + data/iso9660.xorriso.README \ data/lslR.1.spaces.args \ data/lslR.1.spaces.input \ data/lslR.1.spaces.output \ diff --git a/tests/src/vfs/extfs/helpers-list/data/iso9660.xorriso.README b/tests/src/vfs/extfs/helpers-list/data/iso9660.xorriso.README new file mode 100644 index 000000000..138f047c5 --- /dev/null +++ b/tests/src/vfs/extfs/helpers-list/data/iso9660.xorriso.README @@ -0,0 +1,49 @@ +Sample iso image was created from alpine-standard-3.19.0-x86_64.iso with such +script: + + #!/usr/bin/env bash + + if ! test -d links; then + mkdir "$_" + cd "$_" + + touch no-quotes "inner'quote" "'side-quotes'" + + ln -s no-quotes no-quotes-link + ln -s no-quotes "inner'quote-link-nq" + ln -s no-quotes "'side-quotes-link-nq'" + ln -s "inner'quote" "inner'quote-link-iq" + ln -s "inner'quote" "'side-quotes-link-iq'" + ln -s "'side-quotes'" "inner'quote-link-sq" + ln -s "'side-quotes'" "'side-quotes-link-sq'" + + cd - + fi + + args=( + -indev alpine-standard-3.19.0-x86_64.iso + -outdev sample.iso + -boot_image any replay + -joliet on + -rm_r /apks -- + -cpr links / -- + -chown_r 0 /links -- + -chgrp_r 0 /links -- + ) + + xorriso "${args[@]}" + +Features: +- boot catalog file +- subdirs +- files with q(') included +- links with q(') included (name, target, both) + +The quoted stuff is here because xorriso wraps pathes in single quotes and +escapes internal quotes, all with misc_funct.c:Text_shellsafe(). We revert it +back in the helper code. + + +Sample input was generated like this: + + xorriso -dev sample.iso -find / -exec lsdl 2>/dev/null diff --git a/tests/src/vfs/extfs/helpers-list/data/iso9660.xorriso.env_vars b/tests/src/vfs/extfs/helpers-list/data/iso9660.xorriso.env_vars new file mode 100644 index 000000000..b3674e5bf --- /dev/null +++ b/tests/src/vfs/extfs/helpers-list/data/iso9660.xorriso.env_vars @@ -0,0 +1 @@ +MC_TEST_EXTFS_ISO9660_TOOL=xorriso diff --git a/tests/src/vfs/extfs/helpers-list/data/iso9660.xorriso.input b/tests/src/vfs/extfs/helpers-list/data/iso9660.xorriso.input new file mode 100644 index 000000000..05548fc60 --- /dev/null +++ b/tests/src/vfs/extfs/helpers-list/data/iso9660.xorriso.input @@ -0,0 +1,35 @@ +dr-xr-xr-x 1 0 0 0 Dec 7 11:48 '/' +-r--r--r-- 1 0 0 30 Dec 7 11:48 '/.alpine-release' +dr-xr-xr-x 1 0 0 0 Dec 7 11:48 '/boot' +-r--r--r-- 1 0 0 5751847 Dec 7 11:48 '/boot/System.map-lts' +-r--r--r-- 1 0 0 261013 Dec 7 11:48 '/boot/config-lts' +dr-xr-xr-x 1 0 0 0 Dec 7 11:48 '/boot/dtbs-lts' +dr-xr-xr-x 1 0 0 0 Dec 7 11:48 '/boot/grub' +-r--r--r-- 1 0 0 1474560 Dec 4 23:01 '/boot/grub/efi.img' +-r--r--r-- 1 0 0 140 Dec 7 11:48 '/boot/grub/grub.cfg' +-r--r--r-- 1 0 0 20486095 Dec 7 11:48 '/boot/initramfs-lts' +-r--r--r-- 1 0 0 160346112 Dec 7 11:48 '/boot/modloop-lts' +dr-xr-xr-x 1 0 0 0 Dec 7 11:48 '/boot/syslinux' +er--r--r-- 1 0 0 2048 Dec 4 23:01 '/boot/syslinux/boot.cat' +-r--r--r-- 1 0 0 432 May 16 2023 '/boot/syslinux/isohdpfx.bin' +-r--r--r-- 1 0 0 43008 May 16 2023 '/boot/syslinux/isolinux.bin' +-r--r--r-- 1 0 0 115468 May 16 2023 '/boot/syslinux/ldlinux.c32' +-r--r--r-- 1 0 0 178532 May 16 2023 '/boot/syslinux/libcom32.c32' +-r--r--r-- 1 0 0 23636 May 16 2023 '/boot/syslinux/libutil.c32' +-r--r--r-- 1 0 0 11712 May 16 2023 '/boot/syslinux/mboot.c32' +-r--r--r-- 1 0 0 193 Dec 7 11:48 '/boot/syslinux/syslinux.cfg' +-r--r--r-- 1 0 0 11879168 Dec 7 11:48 '/boot/vmlinuz-lts' +dr-xr-xr-x 1 0 0 0 Dec 7 11:48 '/efi' +dr-xr-xr-x 1 0 0 0 Dec 7 11:48 '/efi/boot' +-r--r--r-- 1 0 0 688128 Dec 7 11:48 '/efi/boot/bootx64.efi' +drwxr-xr-x 1 0 0 0 May 7 01:28 '/links' +-rw-r--r-- 1 0 0 0 May 7 01:28 '/links/'"'"'side-quotes'"'"'' +lrwxrwxrwx 1 0 0 0 May 7 01:28 '/links/'"'"'side-quotes-link-iq'"'"'' -> 'inner'"'"'quote' +lrwxrwxrwx 1 0 0 0 May 7 01:28 '/links/'"'"'side-quotes-link-nq'"'"'' -> 'no-quotes' +lrwxrwxrwx 1 0 0 0 May 7 01:28 '/links/'"'"'side-quotes-link-sq'"'"'' -> ''"'"'side-quotes'"'"'' +-rw-r--r-- 1 0 0 0 May 7 01:28 '/links/inner'"'"'quote' +lrwxrwxrwx 1 0 0 0 May 7 01:28 '/links/inner'"'"'quote-link-iq' -> 'inner'"'"'quote' +lrwxrwxrwx 1 0 0 0 May 7 01:28 '/links/inner'"'"'quote-link-nq' -> 'no-quotes' +lrwxrwxrwx 1 0 0 0 May 7 01:28 '/links/inner'"'"'quote-link-sq' -> ''"'"'side-quotes'"'"'' +-rw-r--r-- 1 0 0 0 May 7 01:28 '/links/no-quotes' +lrwxrwxrwx 1 0 0 0 May 7 01:28 '/links/no-quotes-link' -> 'no-quotes' diff --git a/tests/src/vfs/extfs/helpers-list/data/iso9660.xorriso.output b/tests/src/vfs/extfs/helpers-list/data/iso9660.xorriso.output new file mode 100644 index 000000000..3e21c1430 --- /dev/null +++ b/tests/src/vfs/extfs/helpers-list/data/iso9660.xorriso.output @@ -0,0 +1,34 @@ +-r--r--r-- 1 0 0 30 2023-12-07 11:48:00 /.alpine-release +dr-xr-xr-x 1 0 0 0 2023-12-07 11:48:00 /boot +-r--r--r-- 1 0 0 5751847 2023-12-07 11:48:00 /boot/System.map-lts +-r--r--r-- 1 0 0 261013 2023-12-07 11:48:00 /boot/config-lts +dr-xr-xr-x 1 0 0 0 2023-12-07 11:48:00 /boot/dtbs-lts +dr-xr-xr-x 1 0 0 0 2023-12-07 11:48:00 /boot/grub +-r--r--r-- 1 0 0 1474560 2023-12-04 23:01:00 /boot/grub/efi.img +-r--r--r-- 1 0 0 140 2023-12-07 11:48:00 /boot/grub/grub.cfg +-r--r--r-- 1 0 0 20486095 2023-12-07 11:48:00 /boot/initramfs-lts +-r--r--r-- 1 0 0 160346112 2023-12-07 11:48:00 /boot/modloop-lts +dr-xr-xr-x 1 0 0 0 2023-12-07 11:48:00 /boot/syslinux +-r--r--r-- 1 0 0 2048 2023-12-04 23:01:00 /boot/syslinux/boot.cat +-r--r--r-- 1 0 0 432 2023-05-16 00:00:00 /boot/syslinux/isohdpfx.bin +-r--r--r-- 1 0 0 43008 2023-05-16 00:00:00 /boot/syslinux/isolinux.bin +-r--r--r-- 1 0 0 115468 2023-05-16 00:00:00 /boot/syslinux/ldlinux.c32 +-r--r--r-- 1 0 0 178532 2023-05-16 00:00:00 /boot/syslinux/libcom32.c32 +-r--r--r-- 1 0 0 23636 2023-05-16 00:00:00 /boot/syslinux/libutil.c32 +-r--r--r-- 1 0 0 11712 2023-05-16 00:00:00 /boot/syslinux/mboot.c32 +-r--r--r-- 1 0 0 193 2023-12-07 11:48:00 /boot/syslinux/syslinux.cfg +-r--r--r-- 1 0 0 11879168 2023-12-07 11:48:00 /boot/vmlinuz-lts +dr-xr-xr-x 1 0 0 0 2023-12-07 11:48:00 /efi +dr-xr-xr-x 1 0 0 0 2023-12-07 11:48:00 /efi/boot +-r--r--r-- 1 0 0 688128 2023-12-07 11:48:00 /efi/boot/bootx64.efi +drwxr-xr-x 1 0 0 0 2024-05-07 01:28:00 /links +-rw-r--r-- 1 0 0 0 2024-05-07 01:28:00 /links/'side-quotes' +lrwxrwxrwx 1 0 0 0 2024-05-07 01:28:00 /links/'side-quotes-link-iq' -> inner'quote +lrwxrwxrwx 1 0 0 0 2024-05-07 01:28:00 /links/'side-quotes-link-nq' -> no-quotes +lrwxrwxrwx 1 0 0 0 2024-05-07 01:28:00 /links/'side-quotes-link-sq' -> 'side-quotes' +-rw-r--r-- 1 0 0 0 2024-05-07 01:28:00 /links/inner'quote +lrwxrwxrwx 1 0 0 0 2024-05-07 01:28:00 /links/inner'quote-link-iq -> inner'quote +lrwxrwxrwx 1 0 0 0 2024-05-07 01:28:00 /links/inner'quote-link-nq -> no-quotes +lrwxrwxrwx 1 0 0 0 2024-05-07 01:28:00 /links/inner'quote-link-sq -> 'side-quotes' +-rw-r--r-- 1 0 0 0 2024-05-07 01:28:00 /links/no-quotes +lrwxrwxrwx 1 0 0 0 2024-05-07 01:28:00 /links/no-quotes-link -> no-quotes