import requests from bs4 import BeautifulSoup import re from multiprocessing import Pool import itertools base = "https://repo.radeon.com/rocm/apt" r = requests.get(base).text versionPage = BeautifulSoup(r, 'html.parser') getVersion = lambda e : e.get('href')[:-1] versions = list(map(getVersion, versionPage.findAll('a', attrs={'href': lambda v: re.match(r'^\d', v)}))) def checkVersion(v: str): r = requests.get(f"{base}/{v}/dists").text distPage = BeautifulSoup(r, 'html.parser') codes = distPage.findAll('a', attrs={'href': lambda v: not re.match(r'^[http|\.{2}]', v)}) return [(v, code.string[:-1]) for code in codes] with Pool() as p: results = p.map(checkVersion, versions) for r in results: for (v, c) in r: print(f"{v}\t{c}")