root/egg/pypi.py

#!/usr/bin/env python
# encoding: utf-8
"""PythonEggs PyPI communication.
"""

import time, xmlrpclib
from cache import Egg

# needsync = True => needs to be refetched
# updated = when was the entry updated on disk
# timestamp = int(time.time())


def sync(store):
    pypi_xmlrpc = xmlrpclib.Server('http://pypi.python.org/pypi')
    # http://wiki.python.org/moin/CheeseShopXmlRpc

    now = int(time.time())

    # get and update package list
    cached_packages = set([ egg.name for egg in store.find(Egg, source=u"pypi") ])
    packages = set([ unicode(p) for p in pypi_xmlrpc.list_packages() ])
    
    # delete packages no longer on pypi
    for package_name in cached_packages - packages:
        print "removing", package_name
        store.find(Egg, name=package_name, source=u"pypi").remove()

    # add new packages
    for package_name in packages - cached_packages:
        print "adding", package_name
        store.add(Egg(name=unicode(package_name), source=u"pypi", needsync=True))

    store.commit()
    
    # find last successful update
    (last,) = store.execute("SELECT max(updated) FROM egg").get_one()
    
    # find updated packages, mark them as "missing information"
    if last:
        # changelog(since) -> (name, version, timestamp, action)
        for (package_name, version, timestamp, action) in pypi_xmlrpc.changelog(last):
            version = unicode(version)
            package_name = unicode(package_name)
            print "invalidating cache for %s (%s)" % (package_name, action)
            for egg in store.find(Egg, name=package_name, source=u"pypi"):
                egg.needsync = True
                egg.updated = timestamp
        store.commit()

    # update packages with missing information
    # by convention we let all incomplete packages have fullname=None
    incomplete = [ egg.name for egg in store.find(Egg, source=u"pypi", needsync=True) ]
    for num, package_name in enumerate(sorted(incomplete)):
        print "updating %d of %d - %s" % (num+1, len(incomplete), package_name),

        releases = pypi_xmlrpc.package_releases(package_name)
        if not releases:
            print "- warning! no releases"
            # the package is empty; will ignore and rely on changelog to
            # provide updates for it
            
            egg.needssync = False
            continue
        
        # remove old release, replacing with current
        store.find(Egg, name=package_name, source=u"pypi").remove()
        store.commit()

        for version in releases:
            #download_urls = pypi_xmlrpc.release_urls(package_name, version)
            # ({'url': '...', 'packagetype': 'sdist/bdist'}, ...)
            data = pypi_xmlrpc.release_data(package_name, version) # dict ...
            print "- %s" % (version,),
            e = Egg(
                installed = False,
                active = False,
                name = unicode(package_name),
                fullname = unicode(data['name']),
                location = unicode("http://pypi.python.org/pypi/%s" % (package_name,)),
                version = unicode(version),
                classifiers = unicode(data['classifiers']),
                keywords = unicode(data['keywords']),
                description = unicode(data['description']),
                summary = unicode(data['summary']),
                maintainer = unicode(data['maintainer']),
                maintainer_email = unicode(data['maintainer_email']),
                author = unicode(data['author']),
                author_email = unicode(data['author_email']),
                download_url = unicode(data['download_url']),
                platform = unicode(data['platform']),
                obsoletes = unicode(data['obsoletes']),
                provides = unicode(data['provides']),
                requires = unicode(data['requires']),
                license = unicode(data['license']),
                home_page = unicode(data['home_page']),
                stable_version = unicode(data['stable_version']),
                cheesecake_data = data or u"",
                updated = now,
                source = u"pypi",
            )
            store.add(e)
        print
        store.commit()