#!/usr/bin/env python
import requests
import lxml
import gevent
from lxml import html
import os
from gevent import monkey; monkey.patch_os()
monkey.patch_socket()
from multiprocessing import Process
#view-source:https://repos.fedorapeople.org/repos/openstack/openstack-icehouse/epel-6/epel/
base_url = "https://repos.fedorapeople.org/repos/openstack/openstack-icehouse/epel-6/epel/"
url_send="https://repos.fedorapeople.org/repos/openstack/openstack-icehouse/epel-6/"
repodata_url ="https://repos.fedorapeople.org/repos/openstack/openstack-icehouse/epel-6/repodata/"
puppetlab = "http://
yum.puppetlabs.com/el/6/products/x86_64/"
def get_rpms(url):
r = requests.get( url )
doc = html.fromstring(r.text)
doc.make_links_absolute( url )
links = doc.iterlinks()
rpms = [i[2] for i in links if i[2].endswith("rpm")]
return rpms
def get_gzs(url):
r = requests.get(url)
doc = html.fromstring(r.text)
doc.make_links_absolute(url)
links = doc.iterlinks()
rpms = [i[2] for i in links if i[2].endswith("gz") or i[2].endswith("bz2")]
return rpms
def download(url, dir):
r = requests.get( url )
filename = url.split("/")[-1]
print(filename)
f = open( os.path.join(dir, filename) , "wb")
f.write(r.content)
f.close()
rpms = get_rpms ( puppetlab )
w = []
for i in rpms:
print(i)
g = gevent.spawn(download, i, "puppetlab",)
w.append(g)
gevent.joinall(w)
gevent.sleep(0.3)
