Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update indexes_by_name api: allow to specify env_name to restrict only fetching indexes from that env #107

Merged
merged 2 commits into from
Jan 29, 2024
Merged

Update indexes_by_name api: allow to specify env_name to restrict only fetching indexes from that env #107

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
efa6d2b
Update indexes_by_name api: allow to specify env_name to restrict onl…
sengineer0 Jan 29, 2024
b345241
Add doc_type to indexes_by_name api
sengineer0 Jan 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,3 +41,4 @@ config.py
.notes/
.env
.notes
ssh_host_key*
20 changes: 14 additions & 6 deletions biothings/hub/dataindex/indexer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -318,7 +318,9 @@ def __init__(self, build_doc, indexer_env, index_name):
self.pinfo = ProcessInfo(self, indexer_env.get("concurrency", 10))

def setup_log(self):
log_folder = os.path.join(btconfig.LOG_FOLDER, "build", self.build_name or "", "index") if btconfig.LOG_FOLDER else None
log_folder = (
os.path.join(btconfig.LOG_FOLDER, "build", self.build_name or "", "index") if btconfig.LOG_FOLDER else None
)
log_name = f"index_{self.es_index_name}"
self.logger, self.logfile = get_logger(log_name, log_folder=log_folder, force=True)

Expand Down Expand Up @@ -804,8 +806,9 @@ async def _enhance(conf):

return self._config

def get_indexes_by_name(self, index_name=None, limit=10):
def get_indexes_by_name(self, index_name=None, env_name=None, limit=10):
"""Accept an index_name and return a list of indexes get from all elasticsearch environments
or from specific elasticsearch environment.

If index_name is blank, it will be return all indexes.
limit can be used to specify how many indexes should be return.
Expand All @@ -829,9 +832,13 @@ def get_indexes_by_name(self, index_name=None, limit=10):
index_name = "*"
limit = int(limit)

async def fetch(index_name, limit=None):
async def fetch(index_name, env_name=None, limit=None):
indexes = []
for env_name, env in self.register.items():
for _env_name, env in self.register.items():
# If env_name is set, only fetch indexes for the specific es server
if env_name and env_name != _env_name:
continue

async with AsyncElasticsearch(**env["args"]) as client:
try:
indices = await client.indices.get(index_name)
Expand All @@ -842,11 +849,12 @@ async def fetch(index_name, limit=None):
indexes.append(
{
"index_name": index_name,
"doc_type": mapping_meta["biothing_type"],
"build_version": mapping_meta["build_version"],
"count": mapping_meta["stats"]["total"],
"creation_date": index_data["settings"]["index"]["creation_date"],
"environment": {
"name": env_name,
"name": _env_name,
"host": env["args"]["hosts"],
},
}
Expand All @@ -858,7 +866,7 @@ async def fetch(index_name, limit=None):
indexes = indexes[:limit]
return indexes

job = asyncio.ensure_future(fetch(index_name, limit=limit))
job = asyncio.ensure_future(fetch(index_name, env_name=env_name, limit=limit))
job.add_done_callback(self.logger.debug)
return job

Expand Down