diff options
author | 2024-03-03 17:41:08 +0100 | |
---|---|---|
committer | 2024-04-28 01:04:08 +0100 | |
commit | 8c6e5d06afbf6fca1893cff5ed777e44f93b7a5d (patch) | |
tree | 7f64145f2330e10fe8b295bc1f4cd8f2b128fe05 | |
parent | sync: Add method to download zip archives (diff) | |
download | portage-8c6e5d06afbf6fca1893cff5ed777e44f93b7a5d.tar.gz portage-8c6e5d06afbf6fca1893cff5ed777e44f93b7a5d.tar.bz2 portage-8c6e5d06afbf6fca1893cff5ed777e44f93b7a5d.zip |
sync/zipfile: Handle ETag header
Most services add an ETag header and determine whether the locally
cached version of the URL has expired. So we can add ETag processing to
avoid unnecessary downloading and unpacking of the zip archive.
Signed-off-by: Alexey Gladkov <legion@kernel.org>
Signed-off-by: Sam James <sam@gentoo.org>
-rw-r--r-- | lib/portage/sync/modules/zipfile/zipfile.py | 36 |
1 files changed, 29 insertions, 7 deletions
diff --git a/lib/portage/sync/modules/zipfile/zipfile.py b/lib/portage/sync/modules/zipfile/zipfile.py index 1762d2c8f..bb78b3924 100644 --- a/lib/portage/sync/modules/zipfile/zipfile.py +++ b/lib/portage/sync/modules/zipfile/zipfile.py @@ -10,7 +10,7 @@ import tempfile import urllib.request import portage -from portage.util import writemsg_level +from portage.util import writemsg_level, writemsg_stdout from portage.sync.syncbase import SyncBase @@ -31,13 +31,31 @@ class ZipFile(SyncBase): if kwargs: self._kwargs(kwargs) - # initial checkout - zip_uri = self.repo.sync_uri + req = urllib.request.Request(url=self.repo.sync_uri) - with urllib.request.urlopen(zip_uri) as response: - with tempfile.NamedTemporaryFile(delete=False) as tmp_file: - shutil.copyfileobj(response, tmp_file) - zip_file = tmp_file.name + info = portage.grabdict(os.path.join(self.repo.location, ".info")) + if "etag" in info: + req.add_header("If-None-Match", info["etag"][0]) + + try: + with urllib.request.urlopen(req) as response: + with tempfile.NamedTemporaryFile(delete=False) as tmp_file: + shutil.copyfileobj(response, tmp_file) + + zip_file = tmp_file.name + etag = response.headers.get("etag") + + except urllib.error.HTTPError as resp: + if resp.code == 304: + writemsg_stdout(">>> The repository has not changed.\n", noiselevel=-1) + return (os.EX_OK, False) + + writemsg_level( + f"!!! Unable to obtain zip archive: {resp}\n", + noiselevel=-1, + level=logging.ERROR, + ) + return (1, False) if not zipfile.is_zipfile(zip_file): msg = "!!! file is not a zip archive." @@ -77,6 +95,10 @@ class ZipFile(SyncBase): with open(dstpath, "wb") as dstfile: shutil.copyfileobj(srcfile, dstfile) + with open(os.path.join(self.repo.location, ".info"), "w") as infofile: + if etag: + infofile.write(f"etag {etag}\n") + os.unlink(zip_file) return (os.EX_OK, True) |