zeroinstall.injector.fetch

1 """ 2 Downloads feeds, keys, packages and icons. 3 """ 4 5 # Copyright (C) 2008, Thomas Leonard 6 # See the README file for details, or visit http://0install.net. 7 8 import os, sys 9 from logging import info, debug, warn 10 11 from zeroinstall.support import tasks, basedir 12 from zeroinstall.injector.namespaces import XMLNS_IFACE, config_site 13 from zeroinstall.injector.model import DownloadSource, Recipe, SafeException, network_offline, escape 14 from zeroinstall.injector.iface_cache import PendingFeed, ReplayAttack 15 from zeroinstall.injector.handler import NoTrustedKeys

16 17 -def _escape_slashes(path):

18 return path.replace('/', '%23')

19

20 -def _get_feed_dir(feed):

21 """The algorithm from 0mirror.""" 22 if '#' in feed: 23 raise SafeException("Invalid URL '%s'" % feed) 24 scheme, rest = feed.split('://', 1) 25 domain, rest = rest.split('/', 1) 26 for x in [scheme, domain, rest]: 27 if not x or x.startswith(','): 28 raise SafeException("Invalid URL '%s'" % feed) 29 return os.path.join('feeds', scheme, domain, _escape_slashes(rest))

30

31 -class Fetcher(object):

32 """Downloads and stores various things. 33 @ivar handler: handler to use for user-interaction 34 @type handler: L{handler.Handler} 35 @ivar feed_mirror: the base URL of a mirror site for keys and feeds 36 @type feed_mirror: str 37 """ 38 __slots__ = ['handler', 'feed_mirror'] 39

40 - def __init__(self, handler):

41 self.handler = handler 42 self.feed_mirror = "http://roscidus.com/0mirror"

43 44 @tasks.async

45 - def cook(self, required_digest, recipe, stores, force = False, impl_hint = None):

46 """Follow a Recipe. 47 @param impl_hint: the Implementation this is for (if any) as a hint for the GUI 48 @see: L{download_impl} uses this method when appropriate""" 49 # Maybe we're taking this metaphor too far? 50 51 # Start downloading all the ingredients. 52 downloads = {} # Downloads that are not yet successful 53 streams = {} # Streams collected from successful downloads 54 55 # Start a download for each ingredient 56 blockers = [] 57 for step in recipe.steps: 58 blocker, stream = self.download_archive(step, force = force, impl_hint = impl_hint) 59 assert stream 60 blockers.append(blocker) 61 streams[step] = stream 62 63 while blockers: 64 yield blockers 65 tasks.check(blockers) 66 blockers = [b for b in blockers if not b.happened] 67 68 from zeroinstall.zerostore import unpack 69 70 # Create an empty directory for the new implementation 71 store = stores.stores[0] 72 tmpdir = store.get_tmp_dir_for(required_digest) 73 try: 74 # Unpack each of the downloaded archives into it in turn 75 for step in recipe.steps: 76 stream = streams[step] 77 stream.seek(0) 78 unpack.unpack_archive_over(step.url, stream, tmpdir, step.extract) 79 # Check that the result is correct and store it in the cache 80 store.check_manifest_and_rename(required_digest, tmpdir) 81 tmpdir = None 82 finally: 83 # If unpacking fails, remove the temporary directory 84 if tmpdir is not None: 85 from zeroinstall import support 86 support.ro_rmtree(tmpdir)

87

88 - def get_feed_mirror(self, url):

89 """Return the URL of a mirror for this feed.""" 90 return '%s/%s/latest.xml' % (self.feed_mirror, _get_feed_dir(url))

91

92 - def download_and_import_feed(self, feed_url, iface_cache, force = False):

93 """Download the feed, download any required keys, confirm trust if needed and import. 94 @param feed_url: the feed to be downloaded 95 @type feed_url: str 96 @param iface_cache: cache in which to store the feed 97 @type iface_cache: L{iface_cache.IfaceCache} 98 @param force: whether to abort and restart an existing download""" 99 from download import DownloadAborted 100 101 debug("download_and_import_feed %s (force = %d)", feed_url, force) 102 assert not feed_url.startswith('/') 103 104 primary = self._download_and_import_feed(feed_url, iface_cache, force, use_mirror = False) 105 106 @tasks.named_async("monitor feed downloads for " + feed_url) 107 def wait_for_downloads(primary): 108 # Download just the upstream feed, unless it takes too long... 109 timeout = tasks.TimeoutBlocker(5, 'Mirror timeout') # 5 seconds 110 111 yield primary, timeout 112 tasks.check(timeout) 113 114 try: 115 tasks.check(primary) 116 if primary.happened: 117 return # OK, primary succeeded! 118 # OK, maybe it's just being slow... 119 info("Feed download from %s is taking a long time. Trying mirror too...", feed_url) 120 primary_ex = None 121 except NoTrustedKeys, ex: 122 raise # Don't bother trying the mirror if we have a trust problem 123 except ReplayAttack, ex: 124 raise # Don't bother trying the mirror if we have a replay attack 125 except DownloadAborted, ex: 126 raise # Don't bother trying the mirror if the user cancelled 127 except SafeException, ex: 128 # Primary failed 129 primary = None 130 primary_ex = ex 131 warn("Trying mirror, as feed download from %s failed: %s", feed_url, ex) 132 133 # Start downloading from mirror... 134 mirror = self._download_and_import_feed(feed_url, iface_cache, force, use_mirror = True) 135 136 # Wait until both mirror and primary tasks are complete... 137 while True: 138 blockers = filter(None, [primary, mirror]) 139 if not blockers: 140 break 141 yield blockers 142 143 if primary: 144 try: 145 tasks.check(primary) 146 if primary.happened: 147 primary = None 148 # No point carrying on with the mirror once the primary has succeeded 149 if mirror: 150 info("Primary feed download succeeded; aborting mirror download for " + feed_url) 151 mirror.dl.abort() 152 except SafeException, ex: 153 primary = None 154 primary_ex = ex 155 info("Feed download from %s failed; still trying mirror: %s", feed_url, ex) 156 157 if mirror: 158 try: 159 tasks.check(mirror) 160 if mirror.happened: 161 mirror = None 162 if primary_ex: 163 # We already warned; no need to raise an exception too, 164 # as the mirror download succeeded. 165 primary_ex = None 166 except ReplayAttack, ex: 167 info("Version from mirror is older than cached version; ignoring it: %s", ex) 168 mirror = None 169 primary_ex = None 170 except SafeException, ex: 171 info("Mirror download failed: %s", ex) 172 mirror = None 173 174 if primary_ex: 175 raise primary_ex

176 177 return wait_for_downloads(primary)

178

179 - def _download_and_import_feed(self, feed_url, iface_cache, force, use_mirror):

180 """Download and import a feed. 181 @param use_mirror: False to use primary location; True to use mirror.""" 182 if use_mirror: 183 url = self.get_feed_mirror(feed_url) 184 else: 185 url = feed_url 186 187 dl = self.handler.get_download(url, force = force, hint = feed_url) 188 stream = dl.tempfile 189 190 @tasks.named_async("fetch_feed " + url) 191 def fetch_feed(): 192 yield dl.downloaded 193 tasks.check(dl.downloaded) 194 195 pending = PendingFeed(feed_url, stream) 196 197 if use_mirror: 198 # If we got the feed from a mirror, get the key from there too 199 key_mirror = self.feed_mirror + '/keys/' 200 else: 201 key_mirror = None 202 203 keys_downloaded = tasks.Task(pending.download_keys(self.handler, feed_hint = feed_url, key_mirror = key_mirror), "download keys for " + feed_url) 204 yield keys_downloaded.finished 205 tasks.check(keys_downloaded.finished) 206 207 iface = iface_cache.get_interface(pending.url) 208 if not iface_cache.update_interface_if_trusted(iface, pending.sigs, pending.new_xml): 209 blocker = self.handler.confirm_trust_keys(iface, pending.sigs, pending.new_xml) 210 if blocker: 211 yield blocker 212 tasks.check(blocker) 213 if not iface_cache.update_interface_if_trusted(iface, pending.sigs, pending.new_xml): 214 raise NoTrustedKeys("No signing keys trusted; not importing")

215 216 task = fetch_feed() 217 task.dl = dl 218 return task 219

220 - def download_impl(self, impl, retrieval_method, stores, force = False):

221 """Download an implementation. 222 @param impl: the selected implementation 223 @type impl: L{model.ZeroInstallImplementation} 224 @param retrieval_method: a way of getting the implementation (e.g. an Archive or a Recipe) 225 @type retrieval_method: L{model.RetrievalMethod} 226 @param stores: where to store the downloaded implementation 227 @type stores: L{zerostore.Stores} 228 @param force: whether to abort and restart an existing download 229 @rtype: L{tasks.Blocker}""" 230 assert impl 231 assert retrieval_method 232 233 from zeroinstall.zerostore import manifest 234 alg = impl.id.split('=', 1)[0] 235 if alg not in manifest.algorithms: 236 raise SafeException("Unknown digest algorithm '%s' for '%s' version %s" % 237 (alg, impl.feed.get_name(), impl.get_version())) 238 239 @tasks.async 240 def download_impl(): 241 if isinstance(retrieval_method, DownloadSource): 242 blocker, stream = self.download_archive(retrieval_method, force = force, impl_hint = impl) 243 yield blocker 244 tasks.check(blocker) 245 246 stream.seek(0) 247 self._add_to_cache(stores, retrieval_method, stream) 248 elif isinstance(retrieval_method, Recipe): 249 blocker = self.cook(impl.id, retrieval_method, stores, force, impl_hint = impl) 250 yield blocker 251 tasks.check(blocker) 252 else: 253 raise Exception("Unknown download type for '%s'" % retrieval_method) 254 255 self.handler.impl_added_to_store(impl)

256 return download_impl() 257

258 - def _add_to_cache(self, stores, retrieval_method, stream):

259 assert isinstance(retrieval_method, DownloadSource) 260 required_digest = retrieval_method.implementation.id 261 url = retrieval_method.url 262 stores.add_archive_to_cache(required_digest, stream, retrieval_method.url, retrieval_method.extract, 263 type = retrieval_method.type, start_offset = retrieval_method.start_offset or 0)

264

265 - def download_archive(self, download_source, force = False, impl_hint = None):

266 """Fetch an archive. You should normally call L{download_impl} 267 instead, since it handles other kinds of retrieval method too.""" 268 from zeroinstall.zerostore import unpack 269 270 url = download_source.url 271 if not (url.startswith('http:') or url.startswith('https:') or url.startswith('ftp:')): 272 raise SafeException("Unknown scheme in download URL '%s'" % url) 273 274 mime_type = download_source.type 275 if not mime_type: 276 mime_type = unpack.type_from_url(download_source.url) 277 if not mime_type: 278 raise SafeException("No 'type' attribute on archive, and I can't guess from the name (%s)" % download_source.url) 279 unpack.check_type_ok(mime_type) 280 dl = self.handler.get_download(download_source.url, force = force, hint = impl_hint) 281 dl.expected_size = download_source.size + (download_source.start_offset or 0) 282 return (dl.downloaded, dl.tempfile)

283

284 - def download_icon(self, interface, force = False):

285 """Download an icon for this interface and add it to the 286 icon cache. If the interface has no icon or we are offline, do nothing. 287 @return: the task doing the import, or None 288 @rtype: L{tasks.Task}""" 289 debug("download_icon %s (force = %d)", interface, force) 290 291 # Find a suitable icon to download 292 for icon in interface.get_metadata(XMLNS_IFACE, 'icon'): 293 type = icon.getAttribute('type') 294 if type != 'image/png': 295 debug('Skipping non-PNG icon') 296 continue 297 source = icon.getAttribute('href') 298 if source: 299 break 300 warn('Missing "href" attribute on <icon> in %s', interface) 301 else: 302 info('No PNG icons found in %s', interface) 303 return 304 305 dl = self.handler.get_download(source, force = force, hint = interface) 306 307 @tasks.async 308 def download_and_add_icon(): 309 stream = dl.tempfile 310 yield dl.downloaded 311 try: 312 tasks.check(dl.downloaded) 313 stream.seek(0) 314 315 import shutil 316 icons_cache = basedir.save_cache_path(config_site, 'interface_icons') 317 icon_file = file(os.path.join(icons_cache, escape(interface.uri)), 'w') 318 shutil.copyfileobj(stream, icon_file) 319 except Exception, ex: 320 self.handler.report_error(ex)

321 322 return download_and_add_icon() 323

324 - def download_impls(self, implementations, stores):

325 """Download the given implementations, choosing a suitable retrieval method for each.""" 326 blockers = [] 327 328 to_download = [] 329 for impl in implementations: 330 debug("start_downloading_impls: for %s get %s", impl.feed, impl) 331 source = self.get_best_source(impl) 332 if not source: 333 raise SafeException("Implementation " + impl.id + " of " 334 "interface " + impl.feed.get_name() + " cannot be " 335 "downloaded (no download locations given in " 336 "interface!)") 337 to_download.append((impl, source)) 338 339 for impl, source in to_download: 340 blockers.append(self.download_impl(impl, source, stores))

Source Code for Module zeroinstall.injector.fetch