diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..f6016aa7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,130 @@ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ \ No newline at end of file diff --git a/hub-mirror/hub.py b/hub-mirror/hub.py index 8f79f262..c58b54c6 100644 --- a/hub-mirror/hub.py +++ b/hub-mirror/hub.py @@ -86,29 +86,17 @@ def dynamic_list(self): return self._get_all_repo_names(url) @functools.lru_cache - def _get_all_repo_names(self, url): - page, per_page = 1, 60 - api = url + "?page=0&per_page=" + str(per_page) + def _get_all_repo_names(self, url, page=1): + per_page = 60 + api = url + f"?page={page}&per_page=" + str(per_page) # TODO: src_token support response = self.session.get(api) - # TODO: DRY + all_items = [] if response.status_code != 200: print("Repo getting failed: " + response.text) - return [] + return all_items items = response.json() - all_items = [] - while items: + if items: names = [i['name'] for i in items] - all_items += names - items = None - if 'next' in response.links: - url_next = response.links['next']['url'] - response = self.session.get(url_next) - # TODO: DRY - if response.status_code != 200: - print("Repo getting failed: " + response.text) - return [] - page += 1 - items = response.json() - + return names + self._get_all_repo_names(url, page=page+1) return all_items