Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added untar progress similar to existing unzip #17519

Merged
merged 5 commits into from
Jan 17, 2025
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Restored prev FileProgress location and use with
  • Loading branch information
perseoGI committed Jan 16, 2025
commit b7294e6539b8e446fb0cb1252ba4b542ee40f44f
27 changes: 6 additions & 21 deletions conan/tools/files/files.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,14 @@
import platform
import shutil
import subprocess
import io
from contextlib import contextmanager
from fnmatch import fnmatch
from shutil import which


from conan.api.output import TimedOutput
from conans.client.downloaders.caching_file_downloader import SourcesCachingDownloader
from conan.errors import ConanException
from conans.client.rest.file_uploader import FileProgress
from conans.util.files import rmdir as _internal_rmdir, human_size, check_with_algorithm_sum


Expand Down Expand Up @@ -289,7 +288,7 @@ def unzip(conanfile, filename, destination=".", keep_permissions=False, pattern=
if (filename.endswith(".tar.gz") or filename.endswith(".tgz") or
filename.endswith(".tbz2") or filename.endswith(".tar.bz2") or
filename.endswith(".tar")):
return untargz(filename, destination, pattern, strip_root, extract_filter, output)
return untargz(filename, destination, pattern, strip_root, extract_filter)
if filename.endswith(".gz"):
target_name = filename[:-3] if destination == "." else destination
target_dir = os.path.dirname(target_name)
Expand All @@ -300,12 +299,12 @@ def unzip(conanfile, filename, destination=".", keep_permissions=False, pattern=
shutil.copyfileobj(fin, fout)
return
if filename.endswith(".tar.xz") or filename.endswith(".txz"):
return untargz(filename, destination, pattern, strip_root, extract_filter, output)
return untargz(filename, destination, pattern, strip_root, extract_filter)

import zipfile
full_path = os.path.normpath(os.path.join(os.getcwd(), destination))

with zipfile.ZipFile(FileProgress(filename, msg="Unzipping", mode="r")) as z:
with FileProgress(filename, msg="Unzipping", mode="r") as file, zipfile.ZipFile(file) as z:
zip_info = z.infolist()
if pattern:
zip_info = [zi for zi in zip_info if fnmatch(zi.filename, pattern)]
Expand Down Expand Up @@ -351,10 +350,10 @@ def unzip(conanfile, filename, destination=".", keep_permissions=False, pattern=
output.error(f"Error extract {file_.filename}\n{str(e)}", error_type="exception")
output.writeln("")

def untargz(filename, destination=".", pattern=None, strip_root=False, extract_filter=None, output=None):
def untargz(filename, destination=".", pattern=None, strip_root=False, extract_filter=None):
# NOT EXPOSED at `conan.tools.files` but used in tests
import tarfile
with tarfile.TarFile.open(fileobj=FileProgress(filename, msg="Uncompressing"), mode='r:*') as tarredgzippedFile:
with FileProgress(filename, msg="Uncompressing") as fileobj, tarfile.TarFile.open(fileobj=fileobj, mode='r:*') as tarredgzippedFile:
f = getattr(tarfile, f"{extract_filter}_filter", None) if extract_filter else None
tarredgzippedFile.extraction_filter = f or (lambda member_, _: member_)
if not pattern and not strip_root:
Expand Down Expand Up @@ -534,17 +533,3 @@ def move_folder_contents(conanfile, src_folder, dst_folder):
os.rmdir(src_folder)
except OSError:
pass


class FileProgress(io.FileIO):
def __init__(self, path: str, msg: str = "Uploading", report_interval: float = 10, *args, **kwargs):
super().__init__(path, *args, **kwargs)
self._total_size = os.path.getsize(path)
self._filename = os.path.basename(path)
self._t = TimedOutput(interval=report_interval)
self.msg = msg

def read(self, size: int = -1) -> bytes:
current_percentage = int(self.tell() * 100.0 / self._total_size) if self._total_size != 0 else 0
self._t.info(f"{self.msg} {self._filename}: {current_percentage}%")
return super().read(size)
19 changes: 17 additions & 2 deletions conans/client/rest/file_uploader.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import io
import os
import time

from conan.api.output import ConanOutput
from conan.tools.files.files import FileProgress
from conan.api.output import ConanOutput, TimedOutput
from conans.client.rest import response_to_str
from conan.internal.errors import InternalErrorException, RequestErrorException, AuthenticationException, \
ForbiddenException, NotFoundException
Expand Down Expand Up @@ -93,3 +94,17 @@ def _upload_file(self, url, abs_path, headers, auth, ref):
raise
except Exception as exc:
raise ConanException(exc)


class FileProgress(io.FileIO):
def __init__(self, path: str, msg: str = "Uploading", report_interval: float = 10, *args, **kwargs):
super().__init__(path, *args, **kwargs)
self._total_size = os.path.getsize(path)
self._filename = os.path.basename(path)
self._t = TimedOutput(interval=report_interval)
self.msg = msg

def read(self, size: int = -1) -> bytes:
current_percentage = int(self.tell() * 100.0 / self._total_size) if self._total_size != 0 else 0
self._t.info(f"{self.msg} {self._filename}: {current_percentage}%")
return super().read(size)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It might be better to accumulate the read bytes and avoid the extra self.tell() call

Copy link
Contributor Author

@perseoGI perseoGI Jan 16, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes that is a good one!

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done in the last commit