tools/mpremote: Add mpremote mip install to install packages.

This supports the same package sources as the new `mip` tool.
 - micropython-lib (by name)
 - http(s) & github packages with json description
 - directly downloading a .py/.mpy file

The version is specified with an optional `@version` on the end of the
package name. The target dir, index, and mpy/no-mpy can be set through
command line args.

This work was funded through GitHub Sponsors.

Signed-off-by: Jim Mussared <jim.mussared@gmail.com>

docs/reference/mpremote.rst

@@ -146,6 +146,14 @@ The full list of supported commands are:
   variable ``$EDITOR``). If the editor exits successfully, the updated file will
   be copied back to the device.
 
+- install packages from :term:`micropython-lib` (or GitHub) using the ``mip`` tool:
+
+  .. code-block:: bash
+
+      $ mpremote mip install <packages...>
+
+  See :ref:`packages` for more information.
+
 - mount the local directory on the remote device:
 
   .. code-block:: bash
@@ -269,3 +277,9 @@ Examples
   mpremote cp -r dir/ :
 
   mpremote cp a.py b.py : + repl
+
+  mpremote mip install aioble
+
+  mpremote mip install github:org/repo@branch
+
+  mpremote mip install --target /flash/third-party functools

docs/reference/packages.rst

@@ -78,17 +78,17 @@ The :term:`mpremote` tool also includes the same functionality as ``mip`` and
 can be used from a host PC to install packages to a locally connected device
 (e.g. via USB or UART)::
 
-    $ mpremote install pkgname
+    $ mpremote mip install pkgname
-    $ mpremote install pkgname@x.y
+    $ mpremote mip install pkgname@x.y
-    $ mpremote install http://example.com/x/y/foo.py
+    $ mpremote mip install http://example.com/x/y/foo.py
-    $ mpremote install github:org/repo
+    $ mpremote mip install github:org/repo
-    $ mpremote install github:org/repo@branch-or-tag
+    $ mpremote mip install github:org/repo@branch-or-tag
 
 The ``--target=path``, ``--no-mpy``, and ``--index`` arguments can be set::
 
-    $ mpremote install --target=/flash/third-party pkgname
+    $ mpremote mip install --target=/flash/third-party pkgname
-    $ mpremote install --no-mpy pkgname
+    $ mpremote mip install --no-mpy pkgname
-    $ mpremote install --index https://host/pi pkgname
+    $ mpremote mip install --index https://host/pi pkgname
 
 Installing packages manually
 ----------------------------

tools/mpremote/README.md

@@ -27,6 +27,11 @@ The full list of supported commands are:
                                              --capture <file>
                                              --inject-code <string>
                                              --inject-file <file>
+    mpremote mip install <package...> -- Install packages (from micropython-lib or third-party sources)
+                                         options:
+                                             --target <path>
+                                             --index <url>
+                                             --no-mpy
     mpremote help                     -- print list of commands and exit
 
 Multiple commands can be specified and they will be run sequentially.  Connection
@@ -73,3 +78,5 @@ Examples:
     mpremote cp :main.py .
     mpremote cp main.py :
     mpremote cp -r dir/ :
+    mpremote mip install aioble
+    mpremote mip install github:org/repo@branch

tools/mpremote/mpremote/main.py

@@ -36,6 +36,7 @@ from .commands import (
     do_resume,
     do_soft_reset,
 )
+from .mip import do_mip
 from .repl import do_repl
 
 _PROG = "mpremote"
@@ -162,6 +163,29 @@ def argparse_filesystem():
     return cmd_parser
 
 
+def argparse_mip():
+    cmd_parser = argparse.ArgumentParser(
+        description="install packages from micropython-lib or third-party sources"
+    )
+    _bool_flag(cmd_parser, "mpy", "m", True, "download as compiled .mpy files (default)")
+    cmd_parser.add_argument(
+        "--target", type=str, required=False, help="destination direction on the device"
+    )
+    cmd_parser.add_argument(
+        "--index",
+        type=str,
+        required=False,
+        help="package index to use (defaults to micropython-lib)",
+    )
+    cmd_parser.add_argument("command", nargs=1, help="mip command (e.g. install)")
+    cmd_parser.add_argument(
+        "packages",
+        nargs="+",
+        help="list package specifications, e.g. name, name@version, github:org/repo, github:org/repo@branch",
+    )
+    return cmd_parser
+
+
 def argparse_none(description):
     return lambda: argparse.ArgumentParser(description=description)
 
@@ -216,6 +240,10 @@ _COMMANDS = {
         do_filesystem,
         argparse_filesystem,
     ),
+    "mip": (
+        do_mip,
+        argparse_mip,
+    ),
     "help": (
         do_help,
         argparse_none("print help and exit"),

tools/mpremote/mpremote/mip.py

@@ -0,0 +1,191 @@
+# Micropython package installer
+# Ported from micropython-lib/micropython/mip/mip.py.
+# MIT license; Copyright (c) 2022 Jim Mussared
+
+import urllib.error
+import urllib.request
+import json
+import tempfile
+import os
+
+from .commands import CommandError, show_progress_bar
+
+
+_PACKAGE_INDEX = "https://micropython.org/pi/v2"
+_CHUNK_SIZE = 128
+
+
+# This implements os.makedirs(os.dirname(path))
+def _ensure_path_exists(pyb, path):
+    import os
+
+    split = path.split("/")
+
+    # Handle paths starting with "/".
+    if not split[0]:
+        split.pop(0)
+        split[0] = "/" + split[0]
+
+    prefix = ""
+    for i in range(len(split) - 1):
+        prefix += split[i]
+        if not pyb.fs_exists(prefix):
+            pyb.fs_mkdir(prefix)
+        prefix += "/"
+
+
+# Copy from src (stream) to dest (function-taking-bytes)
+def _chunk(src, dest, length=None, op="downloading"):
+    buf = memoryview(bytearray(_CHUNK_SIZE))
+    total = 0
+    if length:
+        show_progress_bar(0, length, op)
+    while True:
+        n = src.readinto(buf)
+        if n == 0:
+            break
+        dest(buf if n == _CHUNK_SIZE else buf[:n])
+        total += n
+        if length:
+            show_progress_bar(total, length, op)
+
+
+def _rewrite_url(url, branch=None):
+    if not branch:
+        branch = "HEAD"
+    if url.startswith("github:"):
+        url = url[7:].split("/")
+        url = (
+            "https://raw.githubusercontent.com/"
+            + url[0]
+            + "/"
+            + url[1]
+            + "/"
+            + branch
+            + "/"
+            + "/".join(url[2:])
+        )
+    return url
+
+
+def _download_file(pyb, url, dest):
+    try:
+        with urllib.request.urlopen(url) as src:
+            fd, path = tempfile.mkstemp()
+            try:
+                print("Installing:", dest)
+                with os.fdopen(fd, "wb") as f:
+                    _chunk(src, f.write, src.length)
+                _ensure_path_exists(pyb, dest)
+                pyb.fs_put(path, dest, progress_callback=show_progress_bar)
+            finally:
+                os.unlink(path)
+    except urllib.error.HTTPError as e:
+        if e.status == 404:
+            raise CommandError(f"File not found: {url}")
+        else:
+            raise CommandError(f"Error {e.status} requesting {url}")
+    except urllib.error.URLError as e:
+        raise CommandError(f"{e.reason} requesting {url}")
+
+
+def _install_json(pyb, package_json_url, index, target, version, mpy):
+    try:
+        with urllib.request.urlopen(_rewrite_url(package_json_url, version)) as response:
+            package_json = json.load(response)
+    except urllib.error.HTTPError as e:
+        if e.status == 404:
+            raise CommandError(f"Package not found: {package_json_url}")
+        else:
+            raise CommandError(f"Error {e.status} requesting {package_json_url}")
+    except urllib.error.URLError as e:
+        raise CommandError(f"{e.reason} requesting {package_json_url}")
+    for target_path, short_hash in package_json.get("hashes", ()):
+        fs_target_path = target + "/" + target_path
+        file_url = f"{index}/file/{short_hash[:2]}/{short_hash}"
+        _download_file(pyb, file_url, fs_target_path)
+    for target_path, url in package_json.get("urls", ()):
+        fs_target_path = target + "/" + target_path
+        _download_file(pyb, _rewrite_url(url, version), fs_target_path)
+    for dep, dep_version in package_json.get("deps", ()):
+        _install_package(pyb, dep, index, target, dep_version, mpy)
+
+
+def _install_package(pyb, package, index, target, version, mpy):
+    if (
+        package.startswith("http://")
+        or package.startswith("https://")
+        or package.startswith("github:")
+    ):
+        if package.endswith(".py") or package.endswith(".mpy"):
+            print(f"Downloading {package} to {target}")
+            _download_file(
+                pyb, _rewrite_url(package, version), target + "/" + package.rsplit("/")[-1]
+            )
+            return
+        else:
+            if not package.endswith(".json"):
+                if not package.endswith("/"):
+                    package += "/"
+                package += "package.json"
+            print(f"Installing {package} to {target}")
+    else:
+        if not version:
+            version = "latest"
+        print(f"Installing {package} ({version}) from {index} to {target}")
+
+        mpy_version = "py"
+        if mpy:
+            pyb.exec("import sys")
+            mpy_version = (
+                int(pyb.eval("getattr(sys.implementation, '_mpy', 0) & 0xFF").decode()) or "py"
+            )
+
+        package = f"{index}/package/{mpy_version}/{package}/{version}.json"
+
+    _install_json(pyb, package, index, target, version, mpy)
+
+
+def do_mip(state, args):
+    state.did_action()
+
+    if args.command[0] == "install":
+        state.ensure_raw_repl()
+
+        for package in args.packages:
+            version = None
+            if "@" in package:
+                package, version = package.split("@")
+
+            print("Install", package)
+
+            if args.index is None:
+                args.index = _PACKAGE_INDEX
+
+            if args.target is None:
+                state.pyb.exec("import sys")
+                lib_paths = (
+                    state.pyb.eval("'\\n'.join(p for p in sys.path if p.endswith('/lib'))")
+                    .decode()
+                    .split("\n")
+                )
+                if lib_paths and lib_paths[0]:
+                    args.target = lib_paths[0]
+                else:
+                    raise CommandError(
+                        "Unable to find lib dir in sys.path, use --target to override"
+                    )
+
+            if args.mpy is None:
+                args.mpy = True
+
+            try:
+                _install_package(
+                    state.pyb, package, args.index.rstrip("/"), args.target, version, args.mpy
+                )
+            except CommandError:
+                print("Package may be partially installed")
+                raise
+            print("Done")
+    else:
+        raise CommandError(f"mip: '{args.command[0]}' is not a command")

tools/pyboard.py

@@ -476,6 +476,13 @@ class Pyboard:
         t = str(self.eval("pyb.RTC().datetime()"), encoding="utf8")[1:-1].split(", ")
         return int(t[4]) * 3600 + int(t[5]) * 60 + int(t[6])
 
+    def fs_exists(self, src):
+        try:
+            self.exec_("import uos\nuos.stat(%s)" % (("'%s'" % src) if src else ""))
+            return True
+        except PyboardError:
+            return False
+
     def fs_ls(self, src):
         cmd = (
             "import uos\nfor f in uos.ilistdir(%s):\n"